Support lwan web server

The changes include:
Add patches for lwan.
Add script to CMakelist.txt to link library.
Add lwan-preload.c wrapper for lwan.
Add send() system call to wrapper.c.
Add demo and update README.md.

When there is only one connection from the browser,
only send() will be called.
When there are  multiple connections, send() will be called
once and then sendfile() will be called.

Close #20

Makefile

@@ -14,19 +14,26 @@ LIGHTY_NAME := lighttpd1.4-lighttpd-1.4.58
 LIGHTY_PATH := downloads/$(LIGHTY_NAME)
 LIGHTY := lighttpd
 
+LWAN_SOURCE := https://github.com/lpereira/lwan/archive/master.zip
+LWAN_NAME := lwan-master
+LWAN_PATH := downloads/$(LWAN_NAME)
+LWAN := lwan
+
 LIBDUMMY_PATH := $(shell find $(shell pwd) -type f -name "libdummy.so") | sed 's_/_\\/_g'
 PWD := $(shell pwd)
 
 OUT := downloads
 
 all: lkm wrapper
 
-.PHONY: $(WRK) $(NGX) $(LIGHTY) config lkm wrapper clean
+.PHONY: $(WRK) $(NGX) $(LIGHTY) $(LWAN) config lkm wrapper clean
 
 ifeq ($(strip $(TARGET)),nginx)
 TARGET = nginx
-else
+else ifeq ($(strip $(TARGET)),lighttpd)
 TARGET = lighttpd
+else
+TARGET = lwan
 endif
 
 config:
@@ -66,6 +73,17 @@ $(LIGHTY):
 	cd $(LIGHTY_PATH) && sudo make install
 	cp -f configs/lighttpd.conf $(LIGHTY_PATH)/src/lighttpd.conf
 
+$(LWAN):
+	@echo "download lwan..."
+	wget $(LWAN_SOURCE)
+	unzip -d $(OUT) master.zip
+	$(RM) master.zip
+	scripts/lwan.sh $(LWAN_PATH)
+	cd $(OUT) && patch -p1 < ../patches/lwan_thread.patch && patch -p1 < ../patches/lwan_main.patch
+	cd $(LWAN_PATH) && mkdir build && cd build && \
+	cmake .. -DCMAKE_BUILD_TYPE=Release && make
+	cp -f configs/lwan.conf $(LWAN_PATH)/lwan.conf
+
 nginx-launch:
 	./downloads/$(NGX_NAME)/objs/nginx
 
@@ -78,6 +96,12 @@ lighttpd-launch:
 lighttpd-esca-launch:
 	LD_PRELOAD=wrapper/wrapper.so ./$(LIGHTY_PATH)/src/lighttpd -D -f $(LIGHTY_PATH)/src/lighttpd.conf
 
+lwan-launch:
+	./downloads/$(LWAN_NAME)/build/src/bin/lwan/lwan -c $(LWAN_PATH)/lwan.conf
+
+lwan-esca-launch:
+	LD_PRELOAD=wrapper/wrapper.so ./downloads/$(LWAN_NAME)/build/src/bin/lwan/lwan -c $(LWAN_PATH)/lwan.conf
+
 lkm: config
 	sudo $(MAKE) -C $@ $(MAKECMDGOALS)
 
@@ -94,6 +118,7 @@ clean:
 	rm -rf $(WRK_PATH)
 	rm -rf $(NGX_PATH)
 	rm -rf $(LIGHTY_PATH)
+	rm -rf $(LWAN_PATH)
 	rm -rf local
 	$(MAKE) -C lkm clean
 	$(MAKE) -C wrapper clean

README.md

@@ -21,9 +21,9 @@ cd ESCA
 ```
 
 ## Build from source
-Compile files under directory `lkm/` and `wrapper/` (The default target is lighttpd)
+Compile files under directory `lkm/` and `wrapper/` (The default target is lwan)
 ```shell
-make TARGET=<nginx | lighttpd>
+make TARGET=<nginx | lighttpd | lwan>
 ```
 
 ### Build adaptation target
@@ -42,6 +42,11 @@ Download and build lighttpd
 make lighttpd
 ```
 
+Download and build lwan
+```shell
+make lwan
+```
+
 ## Testing
 
 ### Launch Nginx
@@ -68,6 +73,18 @@ make load-lkm
 make lighttpd-esca-launch # lighttpd-esca
 ```
 
+### Launch lwan
+Choose either
+```shell
+make lwan-launch # origin lwan
+```
+or
+
+```shell
+make load-lkm
+make lwan-esca-launch # lwan-esca
+```
+
 ### Download workloads
 ```shell
 git submodule init
@@ -76,7 +93,7 @@ git submodule update
 
 ### Benchmarking
 ```shell
-# nginx is at port 8081; lighttpd is at port 3000
+# nginx is at port 8081; lighttpd is at port 3000; lwan is at port 8080
 downloads/wrk-master/wrk -c 50 -d 5s -t 4 http://localhost:8081/a20.html
 ```
 
@@ -89,6 +106,10 @@ Nginx-ESCA led by about 11% over vanilla Nginx.
 
 lighttpd-ESCA led by about 13% over vanilla lighttpd.
 
+![image](assets/lwan-demo.gif)
+
+lwan-ESCA led by about 30% over vanilla lwan.
+
 ## Citation
 
 Please see our [PDP 2022](https://pdp2022.infor.uva.es/) paper, available in the [IEEE Xplore](https://ieeexplore.ieee.org/abstract/document/9756707) digital library, and you can get a [preprint copy](https://eecheng87.github.io/ESCA/main.pdf).

configs/lwan.conf

@@ -0,0 +1,33 @@
+# Timeout in seconds to keep a connection alive.
+keep_alive_timeout = 15
+
+# Set to true to not print any debugging messages. (Only effective in
+# release builds.)
+quiet = false
+
+# Value of "Expires" header. Default is 1 month and 1 week.
+expires = 1M 1w
+
+# Number of I/O threads. Default (0) is number of online CPUs.
+threads = 1
+
+# Disable HAProxy's PROXY protocol by default. Only enable if needed.
+proxy_protocol = false
+
+# Enable straitjacket by default. The `drop_capabilities` option is `true`
+# by default.  Other options may require more privileges.
+straitjacket
+
+listener *:8080
+
+site {
+    serve_files / {
+            # correct path will be replaced by our script
+            path = /tmp/web
+            # When requesting for file.ext, look for a smaller/newer file.ext.gz,
+            # and serve that instead if `Accept-Encoding: gzip` is in the
+            # request headers.
+            serve precompressed files = true
+    }
+}
+

patches/lwan_main.patch

@@ -0,0 +1,11 @@
+--- old/lwan-master/src/bin/lwan/main.c	2022-06-05 08:53:47.000000000 +0800
++++ new/lwan-master/src/bin/lwan/main.c	2022-06-09 21:13:59.257682000 +0800
+@@ -224,7 +224,7 @@
+
+     c = *lwan_get_default_config();
+     c.listener = strdup("*:8080");
+-
++    esca_init();
+     switch (parse_args(argc, argv, &c, root, &sj)) {
+     case ARGS_SERVE_FILES:
+         lwan_status_info("Serving files from %s", root);

patches/lwan_thread.patch

@@ -0,0 +1,19 @@
+--- old/lwan-master/src/lib/lwan-thread.c	2022-06-05 08:53:47.000000000 +0800
++++ new/lwan-master/src/lib/lwan-thread.c	2022-06-09 21:18:31.790748000 +0800
+@@ -924,7 +924,7 @@
+                 break;
+             continue;
+         }
+-
++        batch_start();
+         for (struct epoll_event *event = events; n_fds--; event++) {
+             struct lwan_connection *conn = event->data.ptr;
+
+@@ -959,6 +959,7 @@
+
+         if (created_coros)
+             timeouts_add(t->wheel, &tq.timeout, 1000);
++        batch_flush();
+     }
+
+     pthread_barrier_wait(&lwan->thread.barrier);

scripts/lwan.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+libpath=$(find $(pwd) -type f -name "libshim.so" | sed 's_/_\\/_g')
+webpath=$(readlink --canonicalize web | sed 's_/_\\/_g')
+lwanpath=$1
+# modify lwan
+sed -i "26s/path.*/path = ${webpath}/" configs/lwan.conf
+sed -i "37a add_library(libshim SHARED IMPORTED GLOBAL)\nset_target_properties(libshim PROPERTIES IMPORTED_LOCATION ${libpath})\nlist(APPEND ADDITIONAL_LIBRARIES libshim)" ${lwanpath}/CMakeLists.txt

wrapper/lwan-preload.c

@@ -0,0 +1,99 @@
+/*
+ * Syscall wrapper for Effective System Call Aggregation (ESCA).
+ *
+ * Copyright (c) 2021-2022 National Cheng Kung University, Taiwan.
+ * Authored by Yao Hwang <yaohwang99@gmail.com>
+ */
+
+int close(int fd)
+{
+    if (!in_segment) {
+        real_close = real_close ? real_close : dlsym(RTLD_NEXT, "close");
+        return real_close(fd);
+    }
+
+    batch_num++;
+    int off = global_j << 6;
+
+    btable[off + global_i].sysnum = 3;
+    btable[off + global_i].rstatus = BENTRY_BUSY;
+    btable[off + global_i].nargs = 1;
+    btable[off + global_i].args[0] = fd;
+
+    if (global_i == MAX_TABLE_SIZE - 1) {
+        if (global_j == MAX_THREAD_NUM - 1) {
+            global_j = 0;
+        } else {
+            global_j++;
+        }
+        global_i = 0;
+    } else {
+        global_i++;
+    }
+
+    return 0;
+}
+
+ssize_t sendfile64(int outfd, int infd, off_t *offset, size_t count)
+{
+    if (!in_segment)
+        return real_sendfile(outfd, infd, offset, count);
+
+    batch_num++;
+    int off = global_j << 6;
+
+    btable[off + global_i].sysnum = 40;
+    btable[off + global_i].rstatus = BENTRY_BUSY;
+    btable[off + global_i].nargs = 4;
+    btable[off + global_i].args[0] = outfd;
+    btable[off + global_i].args[1] = infd;
+    btable[off + global_i].args[2] = offset;
+    btable[off + global_i].args[3] = count;
+
+    if (global_i == MAX_TABLE_SIZE - 1) {
+        if (global_j == MAX_THREAD_NUM - 1) {
+            global_j = 0;
+        } else {
+            global_j++;
+        }
+        global_i = 0;
+    } else {
+        global_i++;
+    }
+
+    /* assume always success */
+    return count;
+}
+
+ssize_t send(int sockfd, const void *buf, size_t len, int flags)
+{
+    if (!in_segment)
+        return real_send(sockfd, buf, len, flags);
+
+    batch_num++;
+    int off = global_j << 6;
+
+    btable[off + global_i].sysnum = 44;
+    btable[off + global_i].rstatus = BENTRY_BUSY;
+    btable[off + global_i].nargs = 6;
+    btable[off + global_i].args[0] = sockfd;
+    btable[off + global_i].args[1] = buf;
+    btable[off + global_i].args[2] = len;
+    btable[off + global_i].args[3] = flags;
+    btable[off + global_i].args[4] = NULL;
+    btable[off + global_i].args[5] = 0;
+
+    if (global_i == MAX_TABLE_SIZE - 1) {
+        if (global_j == MAX_THREAD_NUM - 1) {
+            global_j = 0;
+        } else {
+            global_j++;
+        }
+        global_i = 0;
+    } else {
+        global_i++;
+    }
+
+    /* assume always success */
+    return len;
+}

wrapper/preload.h

@@ -34,3 +34,5 @@ typedef long (*shutdown_t)(int fd, int how);
 shutdown_t real_shutdown;
 typedef long (*writev_t)(int fd, const struct iovec *iov, int iovcnt);
 writev_t real_writev;
+typedef long (*send_t)(int sockfd, const void *buf, size_t len, int flags);
+send_t real_send;

wrapper/wrapper.c

@@ -64,6 +64,7 @@ __attribute__((constructor)) static void setup(void)
         real_shutdown ? real_shutdown : dlsym(RTLD_NEXT, "shutdown");
     real_sendfile =
         real_sendfile ? real_sendfile : dlsym(RTLD_NEXT, "sendfile");
-
+    real_send =
+        real_send ? real_send : dlsym(RTLD_NEXT, "send");
     global_i = global_j = 0;
 }