Add support for using kernel implementations of GHASH

This doesn't really improve performance on my Intel CPU (I guess due to the
context switches), but more tests have to be made, in combination with
offloading the AES to the kernel as well, and on different hardware.

6 files changed, 182 insertions, 3 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 29e3ba5..61dcbc6 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -17,6 +17,7 @@ add_executable(fastd
   fastd.c
   config.c
   handshake.c
+  linux_alg.c
   peer.c
   printf.c
   queue.c
diff --git a/src/fastd.c b/src/fastd.c
index 3bbf30f..ec70c6c 100644
--- a/src/fastd.c
+++ b/src/fastd.c
@@ -28,6 +28,7 @@
 
 #include "fastd.h"
 #include "handshake.h"
+#include "linux_alg.h"
 #include "peer.h"
 #include "task.h"
 
@@ -784,6 +785,8 @@ int main(int argc, char *argv[]) {
 
 	init_log(&ctx);
 
+	fastd_linux_alg_init(&ctx);
+
 	if (conf.generate_key) {
 		conf.protocol->generate_key(&ctx);
 		exit(0);
@@ -857,6 +860,8 @@ int main(int argc, char *argv[]) {
 	free(ctx.protocol_state);
 	free(ctx.eth_addr);
 
+	fastd_linux_alg_close(&ctx);
+
 	close_log(&ctx);
 	fastd_config_release(&ctx, &conf);
 
diff --git a/src/fastd.h b/src/fastd.h
index e1c531b..80ec0d2 100644
--- a/src/fastd.h
+++ b/src/fastd.h
@@ -215,6 +215,8 @@ struct _fastd_context {
 	int sockfd;
 	int sock6fd;
 
+	int algfd_ghash;
+
 	size_t eth_addr_size;
 	size_t n_eth_addr;
 	fastd_peer_eth_addr *eth_addr;
diff --git a/src/linux_alg.c b/src/linux_alg.c
new file mode 100644
index 0000000..70f4752
--- /dev/null
+++ b/src/linux_alg.c
@@ -0,0 +1,114 @@
+/*
+  Copyright (c) 2012, Matthias Schiffer <mschiffer@universe-factory.net>
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+    2. Redistributions in binary form must reproduce the above copyright notice,
+       this list of conditions and the following disclaimer in the documentation
+       and/or other materials provided with the distribution.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+
+#include "linux_alg.h"
+
+#include <linux/if_alg.h>
+#include <unistd.h>
+
+
+#ifndef SOL_ALG
+#define SOL_ALG 279
+#endif
+
+
+void fastd_linux_alg_init(fastd_context *ctx) {
+	ctx->algfd_ghash = socket(AF_ALG, SOCK_SEQPACKET, 0);
+	if (ctx->algfd_ghash < 0)
+		goto error_ghash;
+
+	struct sockaddr_alg sa_ghash = {};
+	sa_ghash.salg_family = AF_ALG;
+	strcpy((char*)sa_ghash.salg_type, "hash");
+	strcpy((char*)sa_ghash.salg_name, "ghash");
+	if (bind(ctx->algfd_ghash, (struct sockaddr*)&sa_ghash, sizeof(sa_ghash)) < 0) {
+		close(ctx->algfd_ghash);
+		goto error_ghash;
+	}
+
+	return;
+
+ error_ghash:
+	pr_info(ctx, "no kernel support for GHASH was found, falling back to userspace implementation");
+	ctx->algfd_ghash = -1;
+}
+
+void fastd_linux_alg_close(fastd_context *ctx) {
+	if (ctx->algfd_ghash >= 0)
+		close(ctx->algfd_ghash);
+}
+
+
+int fastd_linux_alg_ghash_init(fastd_context *ctx, uint8_t key[16]) {
+	if (ctx->algfd_ghash < 0)
+		return -1;
+
+	if (setsockopt(ctx->algfd_ghash, SOL_ALG, ALG_SET_KEY, key, 16) < 0) {
+		pr_error_errno(ctx, "fastd_linux_alg_ghash_init: setsockopt");
+		return -1;
+	}
+
+	int ret = accept(ctx->algfd_ghash, NULL, NULL);
+
+	if (ret < 0) {
+		pr_error_errno(ctx, "fastd_linux_alg_ghash_init: accept");
+		return -1;
+	}
+
+	return ret;
+}
+
+bool fastd_linux_alg_ghash(fastd_context *ctx, int fd, uint8_t out[16], const void *data, size_t len) {
+	if (!len)
+		return false;
+
+	const uint8_t *in = data;
+	const uint8_t *end = in+len;
+
+	while (in < end) {
+		int bytes = write(fd, in, end-in);
+		if (bytes < 0) {
+			pr_error_errno(ctx, "fastd_linux_alg_ghash: write");
+			return false;
+		}
+
+		in += bytes;
+	}
+
+	end = out+16;
+
+	while (out < end) {
+		int bytes = read(fd, out, end-out);
+		if (bytes < 0) {
+			pr_error_errno(ctx, "fastd_linux_alg_ghash: read");
+			return false;
+		}
+
+		out += bytes;
+	}
+
+	return true;
+}
diff --git a/src/linux_alg.h b/src/linux_alg.h
new file mode 100644
index 0000000..e10f23a
--- /dev/null
+++ b/src/linux_alg.h
@@ -0,0 +1,39 @@
+/*
+  Copyright (c) 2012, Matthias Schiffer <mschiffer@universe-factory.net>
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+    2. Redistributions in binary form must reproduce the above copyright notice,
+       this list of conditions and the following disclaimer in the documentation
+       and/or other materials provided with the distribution.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+
+#ifndef _FASTD_LINUX_ALG_H_
+#define _FASTD_LINUX_ALG_H_
+
+#include "fastd.h"
+
+
+void fastd_linux_alg_init(fastd_context *ctx);
+void fastd_linux_alg_close(fastd_context *ctx);
+
+int fastd_linux_alg_ghash_init(fastd_context *ctx, uint8_t key[16]);
+bool fastd_linux_alg_ghash(fastd_context *ctx, int fd, uint8_t out[16], const void *data, size_t len);
+
+#endif /* _FASTD_LINUX_ALG_H_ */
diff --git a/src/method_aes128_gcm.c b/src/method_aes128_gcm.c
index 9f2d6b4..86203a2 100644
--- a/src/method_aes128_gcm.c
+++ b/src/method_aes128_gcm.c
@@ -25,7 +25,10 @@
 
 
 #include "fastd.h"
+#include "linux_alg.h"
+
 #include <crypto_stream_aes128ctr.h>
+#include <unistd.h>
 
 
 #define NONCEBYTES 7
@@ -50,6 +53,8 @@ struct _fastd_method_session_state {
 
 	struct timespec receive_last;
 	uint64_t receive_reorder_seen;
+
+	int algfd_ghash;
 };
 
 
@@ -179,6 +184,8 @@ static fastd_method_session_state* method_session_init(fastd_context *ctx, uint8
 		}
 	}
 
+	session->algfd_ghash = fastd_linux_alg_ghash_init(ctx, Hbase[0].b);
+
 	session->send_nonce[0] = initiator ? 3 : 2;
 	session->receive_nonce[0] = initiator ? 0 : 1;
 
@@ -204,6 +211,8 @@ static bool method_session_want_refresh(fastd_context *ctx, fastd_method_session
 
 static void method_session_free(fastd_context *ctx, fastd_method_session_state *session) {
 	if(session) {
+		close(session->algfd_ghash);
+
 		memset(session, 0, sizeof(fastd_method_session_state));
 		free(session);
 	}
@@ -236,7 +245,16 @@ static inline void put_size(block_t *out, size_t len) {
 	out->b[BLOCKBYTES-1] = len << 3;
 }
 
-static inline void ghash(block_t *out, const block_t *blocks, size_t n_blocks, fastd_method_session_state *session) {
+static inline void ghash(fastd_context *ctx, block_t *out, const block_t *blocks, size_t n_blocks, fastd_method_session_state *session) {
+	if (session->algfd_ghash >= 0) {
+		if (fastd_linux_alg_ghash(ctx, session->algfd_ghash, out->b, blocks, n_blocks*BLOCKBYTES))
+			return;
+
+		/* on error */
+		close(session->algfd_ghash);
+		session->algfd_ghash = -1;
+	}
+
 	memset(out, 0, sizeof(block_t));
 
 	int i;
@@ -274,7 +292,7 @@ static bool method_encrypt(fastd_context *ctx, fastd_peer *peer, fastd_method_se
 	put_size(&outblocks[n_blocks], in.len);
 
 	block_t *sig = outblocks-1;
-	ghash(sig, outblocks, n_blocks+1, session);
+	ghash(ctx, sig, outblocks, n_blocks+1, session);
 	xor_a(sig, &stream[0]);
 
 	fastd_buffer_free(in);
@@ -329,7 +347,7 @@ static bool method_decrypt(fastd_context *ctx, fastd_peer *peer, fastd_method_se
 	put_size(&inblocks[n_blocks], in.len);
 
 	block_t sig;
-	ghash(&sig, inblocks, n_blocks+1, session);
+	ghash(ctx, &sig, inblocks, n_blocks+1, session);
 	xor_a(&sig, &stream[0]);
 
 	if (memcmp(&sig, inblocks-1, BLOCKBYTES) != 0) {