From a43064af07d89d503677a47cb01d4b86cf1786b4 Mon Sep 17 00:00:00 2001 From: Casper Kuethe <43839798+Casper64@users.noreply.github.com> Date: Wed, 12 Jul 2023 08:40:16 +0200 Subject: [PATCH] picoev, picohttparser: reimplement in V (#18506) --- cmd/tools/check_os_api_parity.v | 2 - cmd/tools/modules/testing/common.v | 5 +- examples/pico/pico.v | 17 +- vlib/picoev/README.md | 2 +- vlib/picoev/loop_default.c.v | 95 +++++ vlib/picoev/loop_freebsd.c.v | 203 +++++++++++ vlib/picoev/loop_linux.c.v | 139 ++++++++ vlib/picoev/loop_macos.c.v | 203 +++++++++++ vlib/picoev/picoev.v | 387 ++++++++++++--------- vlib/picoev/socket_util.c.v | 143 ++++++++ vlib/picohttpparser/README.md | 3 +- vlib/picohttpparser/misc.v | 84 ++++- vlib/picohttpparser/picohttpparser.v | 499 +++++++++++++++++++++++++-- vlib/picohttpparser/request.v | 117 ++++--- vlib/picohttpparser/response.v | 8 +- vlib/time/time_nix.c.v | 2 +- 16 files changed, 1651 insertions(+), 258 deletions(-) create mode 100644 vlib/picoev/loop_default.c.v create mode 100644 vlib/picoev/loop_freebsd.c.v create mode 100644 vlib/picoev/loop_linux.c.v create mode 100644 vlib/picoev/loop_macos.c.v create mode 100644 vlib/picoev/socket_util.c.v diff --git a/cmd/tools/check_os_api_parity.v b/cmd/tools/check_os_api_parity.v index af3de574a30fc7..032f73d45c0cb1 100644 --- a/cmd/tools/check_os_api_parity.v +++ b/cmd/tools/check_os_api_parity.v @@ -24,8 +24,6 @@ const ( 'crypto.rand', 'os.bare', 'os2', - 'picohttpparser', - 'picoev', 'szip', 'v.eval', ] diff --git a/cmd/tools/modules/testing/common.v b/cmd/tools/modules/testing/common.v index 5ab2de69c39286..334571094d3959 100644 --- a/cmd/tools/modules/testing/common.v +++ b/cmd/tools/modules/testing/common.v @@ -625,9 +625,8 @@ pub fn prepare_test_session(zargs string, folder string, oskipped []string, main continue } $if windows { - // skip pico and process/command examples on windows - if fnormalised.ends_with('examples/pico/pico.v') - || fnormalised.ends_with('examples/process/command.v') { + // skip process/command examples on windows + if fnormalised.ends_with('examples/process/command.v') { continue } } diff --git a/examples/pico/pico.v b/examples/pico/pico.v index 1725268539ab71..3eb537ef1cc8cf 100644 --- a/examples/pico/pico.v +++ b/examples/pico/pico.v @@ -3,7 +3,7 @@ import picoev import picohttpparser const ( - port = 8088 + port = 8089 ) struct Message { @@ -24,21 +24,25 @@ fn hello_response() string { } fn callback(data voidptr, req picohttpparser.Request, mut res picohttpparser.Response) { - if picohttpparser.cmpn(req.method, 'GET ', 4) { - if picohttpparser.cmp(req.path, '/t') { + if req.method == 'GET' { + if req.path == '/t' { res.http_ok() res.header_server() res.header_date() res.plain() res.body(hello_response()) - } else if picohttpparser.cmp(req.path, '/j') { + } else if req.path == '/j' { res.http_ok() res.header_server() res.header_date() res.json() res.body(json_response()) } else { - res.http_404() + res.http_ok() + res.header_server() + res.header_date() + res.html() + res.body('Hello Picoev!\n') } } else { res.http_405() @@ -48,5 +52,6 @@ fn callback(data voidptr, req picohttpparser.Request, mut res picohttpparser.Res fn main() { println('Starting webserver on http://127.0.0.1:${port}/ ...') - picoev.new(port: port, cb: &callback).serve() + mut server := picoev.new(port: port, cb: callback) + server.serve() } diff --git a/vlib/picoev/README.md b/vlib/picoev/README.md index 1934162c2c9b19..f4419cfa986c4f 100644 --- a/vlib/picoev/README.md +++ b/vlib/picoev/README.md @@ -1,4 +1,4 @@ ## Description: -`picoev` is a thin wrapper over [picoev](https://github.com/kazuho/picoev), +`picoev` is a V implementation of [picoev](https://github.com/kazuho/picoev), which in turn is "A tiny, lightning fast event loop for network applications". diff --git a/vlib/picoev/loop_default.c.v b/vlib/picoev/loop_default.c.v new file mode 100644 index 00000000000000..c083deca259fea --- /dev/null +++ b/vlib/picoev/loop_default.c.v @@ -0,0 +1,95 @@ +module picoev + +$if windows { + #include + #include +} $else { + #include +} + +pub struct SelectLoop { +mut: + id int + now i64 +} + +type LoopType = SelectLoop + +// create_select_loop creates a `SelectLoop` struct with `id` +pub fn create_select_loop(id int) !&SelectLoop { + return &SelectLoop{ + id: id + } +} + +[direct_array_access] +fn (mut pv Picoev) update_events(fd int, events int) int { + // check if fd is in range + assert fd < max_fds + + pv.file_descriptors[fd].events = u32(events & picoev_readwrite) + return 0 +} + +[direct_array_access] +fn (mut pv Picoev) poll_once(max_wait int) int { + readfds, writefds, errorfds := C.fd_set{}, C.fd_set{}, C.fd_set{} + + // setup + C.FD_ZERO(&readfds) + C.FD_ZERO(&writefds) + C.FD_ZERO(&errorfds) + + mut maxfd := 0 + + // find the maximum socket for `select` and add sockets to the fd_sets + for target in pv.file_descriptors { + if target.loop_id == pv.loop.id { + if target.events & picoev_read != 0 { + C.FD_SET(target.fd, &readfds) + if maxfd < target.fd { + maxfd = target.fd + } + } + if target.events & picoev_write != 0 { + C.FD_SET(target.fd, &writefds) + if maxfd < target.fd { + maxfd = target.fd + } + } + } + } + + // select and handle sockets if any + tv := C.timeval{ + tv_sec: u64(max_wait) + tv_usec: 0 + } + r := C.@select(maxfd + 1, &readfds, &writefds, &errorfds, &tv) + if r == -1 { + // timeout + return -1 + } else if r > 0 { + for target in pv.file_descriptors { + if target.loop_id == pv.loop.id { + // vfmt off + read_events := ( + (if C.FD_ISSET(target.fd, &readfds) { picoev_read } else { 0 }) + | + (if C.FD_ISSET(target.fd, &writefds) { picoev_write } else { 0 }) + ) + // vfmt on + if read_events != 0 { + $if trace_fd ? { + eprintln('do callback ${target.fd}') + } + + // do callback! + unsafe { target.cb(target.fd, read_events, &pv) } + } + } + } + } + + return 0 +} diff --git a/vlib/picoev/loop_freebsd.c.v b/vlib/picoev/loop_freebsd.c.v new file mode 100644 index 00000000000000..10d6fdddda2cb0 --- /dev/null +++ b/vlib/picoev/loop_freebsd.c.v @@ -0,0 +1,203 @@ +module picoev + +#include +#include +#include + +fn C.kevent(int, changelist voidptr, nchanges int, eventlist voidptr, nevents int, timout &C.timespec) int +fn C.kqueue() int +fn C.EV_SET(kev voidptr, ident int, filter i16, flags u16, fflags u32, data voidptr, udata voidptr) + +pub struct C.kevent { +pub mut: + ident int + // uintptr_t + filter i16 + flags u16 + fflags u32 + data voidptr + // intptr_t + udata voidptr +} + +[heap] +pub struct KqueueLoop { +mut: + id int + now i64 + kq_id int + // -1 if not changed + changed_fds int + events [1024]C.kevent + changelist [256]C.kevent +} + +type LoopType = KqueueLoop + +// create_kqueue_loop creates a new kernel event queue with loop_id=`id` +pub fn create_kqueue_loop(id int) !&KqueueLoop { + mut loop := &KqueueLoop{ + id: id + } + + loop.kq_id = C.kqueue() + if loop.kq_id == -1 { + return error('could not create kqueue loop!') + } + loop.changed_fds = -1 + return loop +} + +// ev_set sets a new `kevent` with file descriptor `index` +[inline] +pub fn (mut pv Picoev) ev_set(index int, operation int, events int) { + // vfmt off + filter := i16( + (if events & picoev_read != 0 { C.EVFILT_READ } else { 0 }) + | + (if events & picoev_write != 0 { C.EVFILT_WRITE } else { 0 }) + ) + // vfmt on + C.EV_SET(&pv.loop.changelist[index], pv.loop.changed_fds, filter, operation, 0, 0, + 0) +} + +// backend_build uses the lower 8 bits to store the old events and the higher 8 +// bits to store the next file descriptor in `Target.backend` +[inline] +fn backend_build(next_fd int, events u32) int { + return int((u32(next_fd) << 8) | (events & 0xff)) +} + +// get the lower 8 bits +[inline] +fn backend_get_old_events(backend int) int { + return backend & 0xff +} + +// get the higher 8 bits +[inline] +fn backend_get_next_fd(backend int) int { + return backend >> 8 +} + +// apply pending processes all changes for the file descriptors and updates `loop.changelist` +// if `aplly_all` is `true` the changes are immediately applied +fn (mut pv Picoev) apply_pending_changes(apply_all bool) int { + mut total, mut nevents := 0, 0 + + for pv.loop.changed_fds != -1 { + mut target := pv.file_descriptors[pv.loop.changed_fds] + old_events := backend_get_old_events(target.backend) + if target.events != old_events { + // events have been changed + if old_events != 0 { + pv.ev_set(total, C.EV_DISABLE, old_events) + total++ + } + if target.events != 0 { + pv.ev_set(total, C.EV_ADD | C.EV_ENABLE, int(target.events)) + total++ + } + // Apply the changes if the total changes exceed the changelist size + if total + 1 >= pv.loop.changelist.len { + nevents = C.kevent(pv.loop.kq_id, &pv.loop.changelist, total, C.NULL, + 0, C.NULL) + assert nevents == 0 + total = 0 + } + } + + pv.loop.changed_fds = backend_get_next_fd(target.backend) + target.backend = -1 + } + + if apply_all && total != 0 { + nevents = C.kevent(pv.loop.kq_id, &pv.loop.changelist, total, C.NULL, 0, C.NULL) + assert nevents == 0 + total = 0 + } + + return total +} + +[direct_array_access] +fn (mut pv Picoev) update_events(fd int, events int) int { + // check if fd is in range + assert fd < max_fds + + mut target := pv.file_descriptors[fd] + + // initialize if adding the fd + if events & picoev_add != 0 { + target.backend = -1 + } + + // return if nothing to do + if (events == picoev_del && target.backend == -1) + || (events != picoev_del && events & picoev_readwrite == target.events) { + return 0 + } + + // add to changed list if not yet being done + if target.backend == -1 { + target.backend = backend_build(pv.loop.changed_fds, target.events) + pv.loop.changed_fds = fd + } + + // update events + target.events = u32(events & picoev_readwrite) + // apply immediately if is a DELETE + if events & picoev_del != 0 { + pv.apply_pending_changes(true) + } + + return 0 +} + +[direct_array_access] +fn (mut pv Picoev) poll_once(max_wait int) int { + ts := C.timespec{ + tv_sec: max_wait + tv_nsec: 0 + } + + mut total, mut nevents := 0, 0 + // apply changes later when the callback is called. + total = pv.apply_pending_changes(false) + + nevents = C.kevent(pv.loop.kq_id, &pv.loop.changelist, total, &pv.loop.events, pv.loop.events.len, + &ts) + if nevents == -1 { + // the errors we can only rescue + assert C.errno == C.EACCES || C.errno == C.EFAULT || C.errno == C.EINTR + return -1 + } + + for i := 0; i < nevents; i++ { + event := pv.loop.events[i] + target := pv.file_descriptors[event.ident] + + // changelist errors are fatal + assert event.flags & C.EV_ERROR == 0 + + if pv.loop.id == target.loop_id && event.filter & (C.EVFILT_READ | C.EVFILT_WRITE) != 0 { + read_events := match int(event.filter) { + C.EVFILT_READ { + picoev_read + } + C.EVFILT_WRITE { + picoev_write + } + else { + 0 + } + } + + // do callback! + unsafe { target.cb(target.fd, read_events, &pv) } + } + } + + return 0 +} diff --git a/vlib/picoev/loop_linux.c.v b/vlib/picoev/loop_linux.c.v new file mode 100644 index 00000000000000..80e026410854aa --- /dev/null +++ b/vlib/picoev/loop_linux.c.v @@ -0,0 +1,139 @@ +module picoev + +#include + +fn C.epoll_create(int) int +fn C.epoll_wait(int, voidptr, int, int) int +fn C.epoll_ctl(int, int, int, voidptr) int + +[typedef] +pub union C.epoll_data_t { +mut: + ptr voidptr + fd int + u32 u32 + u64 u64 +} + +[packed] +pub struct C.epoll_event { +mut: + events u32 + data C.epoll_data_t +} + +[heap] +pub struct EpollLoop { +mut: + id int + epoll_fd int + events [1024]C.epoll_event + now i64 +} + +type LoopType = EpollLoop + +// create_epoll_loop creates a new epoll instance for and returns an +// `EpollLoop` struct with `id` +pub fn create_epoll_loop(id int) !&EpollLoop { + mut loop := &EpollLoop{ + id: id + } + + loop.epoll_fd = C.epoll_create(max_fds) + if loop.epoll_fd == -1 { + return error('could not create epoll loop!') + } + + return loop +} + +[direct_array_access] +fn (mut pv Picoev) update_events(fd int, events int) int { + // check if fd is in range + assert fd < max_fds + + mut target := pv.file_descriptors[fd] + mut ev := C.epoll_event{} + + // fd belongs to loop + if events & picoev_del != target.events && target.loop_id != pv.loop.id { + return -1 + } + + if events & picoev_readwrite == target.events { + return 0 + } + + // vfmt off + ev.events = u32( + (if events & picoev_read != 0 { C.EPOLLIN } else { 0 }) + | + (if events & picoev_write != 0 { C.EPOLLOUT } else { 0 }) + ) + // vfmt on + ev.data.fd = fd + + if events & picoev_del != 0 { + // nothing to do + } else if events & picoev_readwrite == 0 { + // delete the file if it exists + epoll_ret := C.epoll_ctl(pv.loop.epoll_fd, C.EPOLL_CTL_DEL, fd, &ev) + + // check error + assert epoll_ret == 0 + } else { + // change settings to 0 + mut epoll_ret := C.epoll_ctl(pv.loop.epoll_fd, C.EPOLL_CTL_MOD, fd, &ev) + if epoll_ret != 0 { + // if the file is not present we want to add it + assert C.errno == C.ENOENT + epoll_ret = C.epoll_ctl(pv.loop.epoll_fd, C.EPOLL_CTL_ADD, fd, &ev) + + // check error + assert epoll_ret == 0 + } + } + + // convert to u32? + target.events = u32(events) + return 0 +} + +[direct_array_access] +fn (mut pv Picoev) poll_once(max_wait int) int { + nevents := C.epoll_wait(pv.loop.epoll_fd, &pv.loop.events, max_fds, max_wait * 1000) + + if nevents == -1 { + // timeout has occurred + return -1 + } + + for i := 0; i < nevents; i++ { + mut event := pv.loop.events[i] + target := unsafe { pv.file_descriptors[event.data.fd] } + unsafe { + assert event.data.fd < max_fds + } + if pv.loop.id == target.loop_id && target.events & picoev_readwrite != 0 { + // vfmt off + read_events := ( + (if event.events & u32(C.EPOLLIN) != 0 { picoev_read } else { 0 }) + | + (if event.events & u32(C.EPOLLOUT) != 0 { picoev_write } else { 0 }) + ) + // vfmt on + if read_events != 0 { + // do callback! + unsafe { target.cb(event.data.fd, read_events, &pv) } + } + } else { + // defer epoll delete + event.events = 0 + unsafe { + C.epoll_ctl(pv.loop.epoll_fd, C.EPOLL_CTL_DEL, event.data.fd, &event) + } + } + } + return 0 +} diff --git a/vlib/picoev/loop_macos.c.v b/vlib/picoev/loop_macos.c.v new file mode 100644 index 00000000000000..10d6fdddda2cb0 --- /dev/null +++ b/vlib/picoev/loop_macos.c.v @@ -0,0 +1,203 @@ +module picoev + +#include +#include +#include + +fn C.kevent(int, changelist voidptr, nchanges int, eventlist voidptr, nevents int, timout &C.timespec) int +fn C.kqueue() int +fn C.EV_SET(kev voidptr, ident int, filter i16, flags u16, fflags u32, data voidptr, udata voidptr) + +pub struct C.kevent { +pub mut: + ident int + // uintptr_t + filter i16 + flags u16 + fflags u32 + data voidptr + // intptr_t + udata voidptr +} + +[heap] +pub struct KqueueLoop { +mut: + id int + now i64 + kq_id int + // -1 if not changed + changed_fds int + events [1024]C.kevent + changelist [256]C.kevent +} + +type LoopType = KqueueLoop + +// create_kqueue_loop creates a new kernel event queue with loop_id=`id` +pub fn create_kqueue_loop(id int) !&KqueueLoop { + mut loop := &KqueueLoop{ + id: id + } + + loop.kq_id = C.kqueue() + if loop.kq_id == -1 { + return error('could not create kqueue loop!') + } + loop.changed_fds = -1 + return loop +} + +// ev_set sets a new `kevent` with file descriptor `index` +[inline] +pub fn (mut pv Picoev) ev_set(index int, operation int, events int) { + // vfmt off + filter := i16( + (if events & picoev_read != 0 { C.EVFILT_READ } else { 0 }) + | + (if events & picoev_write != 0 { C.EVFILT_WRITE } else { 0 }) + ) + // vfmt on + C.EV_SET(&pv.loop.changelist[index], pv.loop.changed_fds, filter, operation, 0, 0, + 0) +} + +// backend_build uses the lower 8 bits to store the old events and the higher 8 +// bits to store the next file descriptor in `Target.backend` +[inline] +fn backend_build(next_fd int, events u32) int { + return int((u32(next_fd) << 8) | (events & 0xff)) +} + +// get the lower 8 bits +[inline] +fn backend_get_old_events(backend int) int { + return backend & 0xff +} + +// get the higher 8 bits +[inline] +fn backend_get_next_fd(backend int) int { + return backend >> 8 +} + +// apply pending processes all changes for the file descriptors and updates `loop.changelist` +// if `aplly_all` is `true` the changes are immediately applied +fn (mut pv Picoev) apply_pending_changes(apply_all bool) int { + mut total, mut nevents := 0, 0 + + for pv.loop.changed_fds != -1 { + mut target := pv.file_descriptors[pv.loop.changed_fds] + old_events := backend_get_old_events(target.backend) + if target.events != old_events { + // events have been changed + if old_events != 0 { + pv.ev_set(total, C.EV_DISABLE, old_events) + total++ + } + if target.events != 0 { + pv.ev_set(total, C.EV_ADD | C.EV_ENABLE, int(target.events)) + total++ + } + // Apply the changes if the total changes exceed the changelist size + if total + 1 >= pv.loop.changelist.len { + nevents = C.kevent(pv.loop.kq_id, &pv.loop.changelist, total, C.NULL, + 0, C.NULL) + assert nevents == 0 + total = 0 + } + } + + pv.loop.changed_fds = backend_get_next_fd(target.backend) + target.backend = -1 + } + + if apply_all && total != 0 { + nevents = C.kevent(pv.loop.kq_id, &pv.loop.changelist, total, C.NULL, 0, C.NULL) + assert nevents == 0 + total = 0 + } + + return total +} + +[direct_array_access] +fn (mut pv Picoev) update_events(fd int, events int) int { + // check if fd is in range + assert fd < max_fds + + mut target := pv.file_descriptors[fd] + + // initialize if adding the fd + if events & picoev_add != 0 { + target.backend = -1 + } + + // return if nothing to do + if (events == picoev_del && target.backend == -1) + || (events != picoev_del && events & picoev_readwrite == target.events) { + return 0 + } + + // add to changed list if not yet being done + if target.backend == -1 { + target.backend = backend_build(pv.loop.changed_fds, target.events) + pv.loop.changed_fds = fd + } + + // update events + target.events = u32(events & picoev_readwrite) + // apply immediately if is a DELETE + if events & picoev_del != 0 { + pv.apply_pending_changes(true) + } + + return 0 +} + +[direct_array_access] +fn (mut pv Picoev) poll_once(max_wait int) int { + ts := C.timespec{ + tv_sec: max_wait + tv_nsec: 0 + } + + mut total, mut nevents := 0, 0 + // apply changes later when the callback is called. + total = pv.apply_pending_changes(false) + + nevents = C.kevent(pv.loop.kq_id, &pv.loop.changelist, total, &pv.loop.events, pv.loop.events.len, + &ts) + if nevents == -1 { + // the errors we can only rescue + assert C.errno == C.EACCES || C.errno == C.EFAULT || C.errno == C.EINTR + return -1 + } + + for i := 0; i < nevents; i++ { + event := pv.loop.events[i] + target := pv.file_descriptors[event.ident] + + // changelist errors are fatal + assert event.flags & C.EV_ERROR == 0 + + if pv.loop.id == target.loop_id && event.filter & (C.EVFILT_READ | C.EVFILT_WRITE) != 0 { + read_events := match int(event.filter) { + C.EVFILT_READ { + picoev_read + } + C.EVFILT_WRITE { + picoev_write + } + else { + 0 + } + } + + // do callback! + unsafe { target.cb(target.fd, read_events, &pv) } + } + } + + return 0 +} diff --git a/vlib/picoev/picoev.v b/vlib/picoev/picoev.v index 4337012b65d071..0a3eeb8ac0e6ca 100644 --- a/vlib/picoev/picoev.v +++ b/vlib/picoev/picoev.v @@ -1,53 +1,31 @@ -// Copyright (c) 2019-2023 Alexander Medvednikov. All rights reserved. -// Use of this source code is governed by an MIT license -// that can be found in the LICENSE file. module picoev -import net import picohttpparser - -#include -#include -#include -#flag -I @VEXEROOT/thirdparty/picoev -#flag @VEXEROOT/thirdparty/picoev/picoev.o -#include "src/picoev.h" - -[typedef] -struct C.picoev_loop {} - -fn C.picoev_del(&C.picoev_loop, int) int - -fn C.picoev_set_timeout(&C.picoev_loop, int, int) - -// fn C.picoev_handler(loop &C.picoev_loop, fd int, revents int, cb_arg voidptr) -// TODO: (sponge) update to C.picoev_handler with C type def update -type PicoevHandler = fn (loop &C.picoev_loop, fd int, revents int, context voidptr) - -fn C.picoev_add(&C.picoev_loop, int, int, int, &PicoevHandler, voidptr) int - -fn C.picoev_init(int) int - -fn C.picoev_create_loop(int) &C.picoev_loop - -fn C.picoev_loop_once(&C.picoev_loop, int) int - -fn C.picoev_destroy_loop(&C.picoev_loop) int - -fn C.picoev_deinit() int - -const ( - max_fds = 1024 - max_timeout = 10 +import time + +pub const ( + max_fds = 1024 + max_queue = 4096 + + // events + picoev_read = 1 + picoev_write = 2 + picoev_timeout = 4 + picoev_add = 0x40000000 + picoev_del = 0x20000000 + picoev_readwrite = 3 // 1 xor 2 ) -enum Event { - read = C.PICOEV_READ - write = C.PICOEV_WRITE - timeout = C.PICOEV_TIMEOUT - add = C.PICOEV_ADD - del = C.PICOEV_DEL - readwrite = C.PICOEV_READWRITE +// Target is a data representation of everything that needs to be associated with a single +// file descriptor (connection) +pub struct Target { +pub mut: + fd int + loop_id int = -1 + events u32 + cb fn (int, int, voidptr) + // used internally by the kqueue implementation + backend int } pub struct Config { @@ -62,137 +40,230 @@ pub: max_write int = 8192 } -struct Picoev { - loop &C.picoev_loop = unsafe { nil } - cb fn (voidptr, picohttpparser.Request, mut picohttpparser.Response) - err_cb fn (voidptr, picohttpparser.Request, mut picohttpparser.Response, IError) - user_data voidptr +[heap] +pub struct Picoev { + cb fn (voidptr, picohttpparser.Request, mut picohttpparser.Response) + err_cb fn (voidptr, picohttpparser.Request, mut picohttpparser.Response, IError) = default_err_cb + user_data voidptr = unsafe { nil } + timeout_secs int - max_headers int - max_read int - max_write int + max_headers int = 100 + max_read int = 4096 + max_write int = 8192 mut: - date &u8 = unsafe { nil } - buf &u8 = unsafe { nil } - idx [1024]int - out &u8 = unsafe { nil } + loop &LoopType = unsafe { nil } + file_descriptors [max_fds]&Target + timeouts map[int]i64 + num_loops int + + buf &u8 = unsafe { nil } + idx [1024]int + out &u8 = unsafe { nil } + + date string } -[inline] -fn setup_sock(fd int) ! { - flag := 1 - if C.setsockopt(fd, C.IPPROTO_TCP, C.TCP_NODELAY, &flag, sizeof(int)) < 0 { - return error('setup_sock.setup_sock failed') +// init fills the `file_descriptors` array +pub fn (mut pv Picoev) init() { + assert picoev.max_fds > 0 + + pv.num_loops = 0 + + for i in 0 .. picoev.max_fds { + pv.file_descriptors[i] = &Target{} } - $if freebsd { - if C.fcntl(fd, C.F_SETFL, C.SOCK_NONBLOCK) != 0 { - return error('fcntl failed') - } - } $else { - if C.fcntl(fd, C.F_SETFL, C.O_NONBLOCK) != 0 { - return error('fcntl failed') +} + +// add adds a file descriptor to the loop +[direct_array_access] +pub fn (mut pv Picoev) add(fd int, events int, timeout int, cb voidptr) int { + assert fd < picoev.max_fds + + mut target := pv.file_descriptors[fd] + target.fd = fd + target.cb = cb + target.loop_id = pv.loop.id + target.events = 0 + + if pv.update_events(fd, events | picoev.picoev_add) != 0 { + pv.del(fd) + return -1 + } + + // update timeout + pv.set_timeout(fd, timeout) + return 0 +} + +// del removes a file descriptor from the loop +[direct_array_access] +fn (mut pv Picoev) del(fd int) int { + assert fd < picoev.max_fds + mut target := pv.file_descriptors[fd] + + $if trace_fd ? { + eprintln('delete ${fd}') + } + + if pv.update_events(fd, picoev.picoev_del) != 0 { + target.loop_id = -1 + target.fd = 0 + return -1 + } + + pv.set_timeout(fd, 0) + target.loop_id = -1 + target.fd = 0 + return 0 +} + +fn (mut pv Picoev) loop_once(max_wait int) int { + pv.loop.now = get_time() + + if pv.poll_once(max_wait) != 0 { + return -1 + } + + if max_wait != 0 { + pv.loop.now = get_time() + } + + pv.handle_timeout() + return 0 +} + +// set_timeout sets the timeout in seconds for a file descriptor. If a timeout occurs +// the file descriptors target callback is called with a timeout event +[direct_array_access; inline] +fn (mut pv Picoev) set_timeout(fd int, secs int) { + assert fd < picoev.max_fds + if secs != 0 { + pv.timeouts[fd] = pv.loop.now + secs + } else { + pv.timeouts.delete(fd) + } +} + +// handle_timeout loops over all file descriptors and removes them from the loop +// if they are timed out. Also the file descriptors target callback is called with a +// timeout event +[direct_array_access; inline] +fn (mut pv Picoev) handle_timeout() { + for fd, timeout in pv.timeouts { + if timeout <= pv.loop.now { + target := pv.file_descriptors[fd] + assert target.loop_id == pv.loop.id + + pv.timeouts.delete(fd) + unsafe { target.cb(fd, picoev.picoev_timeout, &pv) } } } } -[inline] -fn close_conn(loop &C.picoev_loop, fd int) { - C.picoev_del(voidptr(loop), fd) - C.close(fd) +// accept_callback accepts a new connection from `listen_fd` and adds it to the loop +fn accept_callback(listen_fd int, events int, cb_arg voidptr) { + mut pv := unsafe { &Picoev(cb_arg) } + newfd := accept(listen_fd) + if newfd >= picoev.max_fds { + // should never happen + close_socket(newfd) + return + } + + $if trace_fd ? { + eprintln('accept ${newfd}') + } + + if newfd != -1 { + setup_sock(newfd) or { + eprintln('setup_sock failed, fd: ${newfd}, listen_fd: ${listen_fd}, err: ${err.code()}') + pv.err_cb(pv.user_data, picohttpparser.Request{}, mut &picohttpparser.Response{}, + err) + return + } + pv.add(newfd, picoev.picoev_read, pv.timeout_secs, raw_callback) + } } +// close_conn closes the socket `fd` and removes it from the loop [inline] -fn req_read(fd int, b &u8, max_len int, idx int) int { - unsafe { - return C.read(fd, b + idx, max_len - idx) - } +pub fn (mut pv Picoev) close_conn(fd int) { + pv.del(fd) + close_socket(fd) } -fn rw_callback(loop &C.picoev_loop, fd int, events int, context voidptr) { - mut p := unsafe { &Picoev(context) } +[direct_array_access] +fn raw_callback(fd int, events int, context voidptr) { + mut pv := unsafe { &Picoev(context) } defer { - p.idx[fd] = 0 + pv.idx[fd] = 0 } - if (events & int(Event.timeout)) != 0 { - close_conn(loop, fd) + + if events & picoev.picoev_timeout != 0 { + $if trace_fd ? { + eprintln('timeout ${fd}') + } + pv.close_conn(fd) return - } else if (events & int(Event.read)) != 0 { - C.picoev_set_timeout(voidptr(loop), fd, p.timeout_secs) + } else if events & picoev.picoev_read != 0 { + pv.set_timeout(fd, pv.timeout_secs) - // Request init - mut buf := p.buf + mut buf := pv.buf unsafe { - buf += fd * p.max_read // pointer magic + buf += fd * pv.max_read // pointer magic } mut req := picohttpparser.Request{} // Response init - mut out := p.out + mut out := pv.out unsafe { - out += fd * p.max_write // pointer magic + out += fd * pv.max_write // pointer magic } mut res := picohttpparser.Response{ fd: fd - date: p.date buf_start: out buf: out + date: pv.date.str } for { // Request parsing loop - r := req_read(fd, buf, p.max_read, p.idx[fd]) // Get data from socket + r := req_read(fd, buf, pv.max_read, pv.idx[fd]) // Get data from socket if r == 0 { // connection closed by peer - close_conn(loop, fd) + pv.close_conn(fd) return } else if r == -1 { - // error - if C.errno == C.EAGAIN { - // try again later - return - } - if C.errno == C.EWOULDBLOCK { - // try again later + if fatal_socket_error(fd) == false { return } + // fatal error - close_conn(loop, fd) + pv.close_conn(fd) return } - p.idx[fd] += r + pv.idx[fd] += r - mut s := unsafe { tos(buf, p.idx[fd]) } - pret := req.parse_request(s, p.max_headers) // Parse request via picohttpparser + mut s := unsafe { tos(buf, pv.idx[fd]) } + pret := req.parse_request(s) or { + // Parse error + pv.err_cb(pv.user_data, req, mut &res, err) + return + } if pret > 0 { // Success break - } else if pret == -1 { // Parse error - p.err_cb(p.user_data, req, mut &res, error('ParseError')) - return } assert pret == -2 // request is incomplete, continue the loop - if p.idx[fd] == sizeof(buf) { - p.err_cb(p.user_data, req, mut &res, error('RequestIsTooLongError')) + if pv.idx[fd] == sizeof(buf) { + pv.err_cb(pv.user_data, req, mut &res, error('RequestIsTooLongError')) return } } // Callback (should call .end() itself) - p.cb(p.user_data, req, mut &res) - } -} - -fn accept_callback(loop &C.picoev_loop, fd int, events int, cb_arg voidptr) { - mut p := unsafe { &Picoev(cb_arg) } - newfd := C.accept(fd, 0, 0) - if newfd != -1 { - setup_sock(newfd) or { - p.err_cb(p.user_data, picohttpparser.Request{}, mut &picohttpparser.Response{}, - err) - } - C.picoev_add(voidptr(loop), newfd, int(Event.read), p.timeout_secs, rw_callback, - cb_arg) + pv.cb(pv.user_data, req, mut &res) } } @@ -201,42 +272,12 @@ fn default_err_cb(data voidptr, req picohttpparser.Request, mut res picohttppars res.end() } +// new creates a `Picoev` struct and initializes the main loop pub fn new(config Config) &Picoev { - fd := C.socket(net.AddrFamily.ip, net.SocketType.tcp, 0) - assert fd != -1 + listen_fd := listen(config) - // Setting flags for socket - flag := 1 - assert C.setsockopt(fd, C.SOL_SOCKET, C.SO_REUSEADDR, &flag, sizeof(int)) == 0 - $if linux { - assert C.setsockopt(fd, C.SOL_SOCKET, C.SO_REUSEPORT, &flag, sizeof(int)) == 0 - assert C.setsockopt(fd, C.IPPROTO_TCP, C.TCP_QUICKACK, &flag, sizeof(int)) == 0 - timeout := 10 - assert C.setsockopt(fd, C.IPPROTO_TCP, C.TCP_DEFER_ACCEPT, &timeout, sizeof(int)) == 0 - queue_len := 4096 - assert C.setsockopt(fd, C.IPPROTO_TCP, C.TCP_FASTOPEN, &queue_len, sizeof(int)) == 0 - } - - // Setting addr - mut addr := C.sockaddr_in{ - sin_family: u8(C.AF_INET) - sin_port: C.htons(config.port) - sin_addr: C.htonl(C.INADDR_ANY) - } - size := sizeof(C.sockaddr_in) - bind_res := C.bind(fd, voidptr(unsafe { &net.Addr(&addr) }), size) - assert bind_res == 0 - listen_res := C.listen(fd, C.SOMAXCONN) - assert listen_res == 0 - setup_sock(fd) or { - config.err_cb(config.user_data, picohttpparser.Request{}, mut &picohttpparser.Response{}, - err) - } - - C.picoev_init(picoev.max_fds) - loop := C.picoev_create_loop(picoev.max_timeout) mut pv := &Picoev{ - loop: loop + num_loops: 1 cb: config.cb err_cb: config.err_cb user_data: config.user_data @@ -244,25 +285,45 @@ pub fn new(config Config) &Picoev { max_headers: config.max_headers max_read: config.max_read max_write: config.max_write - date: &u8(C.get_date()) buf: unsafe { malloc_noscan(picoev.max_fds * config.max_read + 1) } out: unsafe { malloc_noscan(picoev.max_fds * config.max_write + 1) } } - C.picoev_add(voidptr(loop), fd, int(Event.read), 0, accept_callback, pv) - spawn update_date(mut pv) + // epoll for linux + // kqueue for macos and bsd + // select for windows and others + $if linux { + pv.loop = create_epoll_loop(0) or { panic(err) } + } $else $if freebsd || macos { + pv.loop = create_kqueue_loop(0) or { panic(err) } + } $else { + pv.loop = create_select_loop(0) or { panic(err) } + } + + pv.init() + + pv.add(listen_fd, picoev.picoev_read, 0, accept_callback) return pv } -pub fn (p Picoev) serve() { +// serve starts the Picoev server +pub fn (mut pv Picoev) serve() { + spawn update_date(mut pv) + for { - C.picoev_loop_once(p.loop, 1) + pv.loop_once(1) } } -fn update_date(mut p Picoev) { +// update_date updates `date` on `pv` every second. +fn update_date(mut pv Picoev) { for { - p.date = &u8(C.get_date()) - C.usleep(1000000) + // get GMT (UTC) time for the HTTP Date header + gmt := time.utc() + mut date := gmt.strftime('---, %d --- %Y %H:%M:%S GMT') + date = date.replace_once('---', gmt.weekday_str()) + date = date.replace_once('---', gmt.smonth()) + pv.date = date + time.sleep(time.second) } } diff --git a/vlib/picoev/socket_util.c.v b/vlib/picoev/socket_util.c.v new file mode 100644 index 00000000000000..63149a894286c1 --- /dev/null +++ b/vlib/picoev/socket_util.c.v @@ -0,0 +1,143 @@ +module picoev + +import net +import picohttpparser + +#include +$if windows { + #include + #include +} $else $if freebsd || macos { + #include + #include + #include + #include +} $else { + #include + #include +} + +[inline] +fn get_time() i64 { + // time.now() is slow + return i64(C.time(C.NULL)) +} + +[inline] +fn accept(fd int) int { + return C.accept(fd, 0, 0) +} + +[inline] +fn close_socket(fd int) { + $if trace_fd ? { + eprintln('close ${fd}') + } + + $if windows { + C.closesocket(fd) + } $else { + C.close(fd) + } +} + +[inline] +fn setup_sock(fd int) ! { + flag := 1 + + if C.setsockopt(fd, C.IPPROTO_TCP, C.TCP_NODELAY, &flag, sizeof(int)) < 0 { + return error('setup_sock.setup_sock failed') + } + + $if freebsd { + if C.fcntl(fd, C.F_SETFL, C.SOCK_NONBLOCK) != 0 { + return error('fcntl failed') + } + } $else $if windows { + non_blocking_mode := u32(1) + if C.ioctlsocket(fd, C.FIONBIO, &non_blocking_mode) == C.SOCKET_ERROR { + return error('icotlsocket failed') + } + } $else { + // linux and macos + if C.fcntl(fd, C.F_SETFL, C.O_NONBLOCK) != 0 { + return error('fcntl failed') + } + } +} + +[inline] +fn req_read(fd int, b &u8, max_len int, idx int) int { + // use `recv` instead of `read` for windows compatibility + unsafe { + return C.recv(fd, b + idx, max_len - idx, 0) + } +} + +fn fatal_socket_error(fd int) bool { + if C.errno == C.EAGAIN { + // try again later + return false + } + $if windows { + if C.errno == C.WSAEWOULDBLOCK { + // try again later + return false + } + } $else { + if C.errno == C.EWOULDBLOCK { + // try again later + return false + } + } + + $if trace_fd ? { + eprintln('fatal error ${fd}: ${C.errno}') + } + + return true +} + +// listen creates a listening tcp socket and returns its file decriptor +fn listen(config Config) int { + // not using the `net` modules sockets, because not all socket options are defined + fd := C.socket(net.AddrFamily.ip, net.SocketType.tcp, 0) + assert fd != -1 + + $if trace_fd ? { + eprintln('listen: ${fd}') + } + + // Setting flags for socket + flag := 1 + assert C.setsockopt(fd, C.SOL_SOCKET, C.SO_REUSEADDR, &flag, sizeof(int)) == 0 + + $if linux { + // epoll socket options + assert C.setsockopt(fd, C.SOL_SOCKET, C.SO_REUSEPORT, &flag, sizeof(int)) == 0 + assert C.setsockopt(fd, C.IPPROTO_TCP, C.TCP_QUICKACK, &flag, sizeof(int)) == 0 + assert C.setsockopt(fd, C.IPPROTO_TCP, C.TCP_DEFER_ACCEPT, &config.timeout_secs, + sizeof(int)) == 0 + queue_len := max_queue + assert C.setsockopt(fd, C.IPPROTO_TCP, C.TCP_FASTOPEN, &queue_len, sizeof(int)) == 0 + } + + // addr settings + mut addr := C.sockaddr_in{ + sin_family: u8(C.AF_INET) + sin_port: C.htons(config.port) + sin_addr: C.htonl(C.INADDR_ANY) + } + size := sizeof(C.sockaddr_in) + bind_res := C.bind(fd, voidptr(unsafe { &net.Addr(&addr) }), size) + assert bind_res == 0 + + listen_res := C.listen(fd, C.SOMAXCONN) + assert listen_res == 0 + setup_sock(fd) or { + config.err_cb(config.user_data, picohttpparser.Request{}, mut &picohttpparser.Response{}, + err) + } + + return fd +} diff --git a/vlib/picohttpparser/README.md b/vlib/picohttpparser/README.md index 5b2c7567532778..79c116d1513f89 100644 --- a/vlib/picohttpparser/README.md +++ b/vlib/picohttpparser/README.md @@ -1,4 +1,5 @@ ## Description: -`picohttpparser` is a thin wrapper over [picohttpparser](https://github.com/h2o/picohttpparser), +`picohttpparser` is V implementation of +[picohttpparser](https://github.com/h2o/picohttpparser), which in turn is "a tiny, primitive, fast HTTP request/response parser." diff --git a/vlib/picohttpparser/misc.v b/vlib/picohttpparser/misc.v index 7bce4e3ecae1e3..a96c5c94490fe2 100644 --- a/vlib/picohttpparser/misc.v +++ b/vlib/picohttpparser/misc.v @@ -1,20 +1,76 @@ module picohttpparser -[inline; unsafe] -fn cpy(dst &u8, src &u8, len int) int { - unsafe { C.memcpy(dst, src, len) } - return len -} +const ( + // vfmt off + g_digits_lut = [ + `0`,`0`,`0`,`1`,`0`,`2`,`0`,`3`,`0`,`4`,`0`,`5`,`0`,`6`,`0`,`7`,`0`,`8`,`0`,`9`, + `1`,`0`,`1`,`1`,`1`,`2`,`1`,`3`,`1`,`4`,`1`,`5`,`1`,`6`,`1`,`7`,`1`,`8`,`1`,`9`, + `2`,`0`,`2`,`1`,`2`,`2`,`2`,`3`,`2`,`4`,`2`,`5`,`2`,`6`,`2`,`7`,`2`,`8`,`2`,`9`, + `3`,`0`,`3`,`1`,`3`,`2`,`3`,`3`,`3`,`4`,`3`,`5`,`3`,`6`,`3`,`7`,`3`,`8`,`3`,`9`, + `4`,`0`,`4`,`1`,`4`,`2`,`4`,`3`,`4`,`4`,`4`,`5`,`4`,`6`,`4`,`7`,`4`,`8`,`4`,`9`, + `5`,`0`,`5`,`1`,`5`,`2`,`5`,`3`,`5`,`4`,`5`,`5`,`5`,`6`,`5`,`7`,`5`,`8`,`5`,`9`, + `6`,`0`,`6`,`1`,`6`,`2`,`6`,`3`,`6`,`4`,`6`,`5`,`6`,`6`,`6`,`7`,`6`,`8`,`6`,`9`, + `7`,`0`,`7`,`1`,`7`,`2`,`7`,`3`,`7`,`4`,`7`,`5`,`7`,`6`,`7`,`7`,`7`,`8`,`7`,`9`, + `8`,`0`,`8`,`1`,`8`,`2`,`8`,`3`,`8`,`4`,`8`,`5`,`8`,`6`,`8`,`7`,`8`,`8`,`8`,`9`, + `9`,`0`,`9`,`1`,`9`,`2`,`9`,`3`,`9`,`4`,`9`,`5`,`9`,`6`,`9`,`7`,`9`,`8`,`9`,`9` + ] + // vfmt on +) -[inline] -pub fn cmp(dst string, src string) bool { - if dst.len != src.len { - return false +// u64toa converts `value` to an ascii string and stores it at `buf_start` +// then it returns the length of the ascii string (branch lookup table implementation) +[direct_array_access; inline] +fn u64toa(buf_start &u8, value u64) !int { + mut buf := unsafe { buf_start } + // set maximum length to 100MB + if value >= 100_000_000 { + return error('Maximum size of 100MB exceeded!') + } + + v := u32(value) + if v < 10_000 { + d1 := u32((v / 100) << 1) + d2 := u32((v % 100) << 1) + unsafe { + if v >= 1000 { + *buf++ = picohttpparser.g_digits_lut[d1] + } + if v >= 100 { + *buf++ = picohttpparser.g_digits_lut[d1 + 1] + } + if v >= 10 { + *buf++ = picohttpparser.g_digits_lut[d2] + } + *buf++ = picohttpparser.g_digits_lut[d2 + 1] + } + } else { + b := v / 10_000 + c := v % 10_000 + + d1 := u32((b / 100) << 1) + d2 := u32((b % 100) << 1) + + d3 := u32((c / 100) << 1) + d4 := u32((c % 100) << 1) + + unsafe { + if value >= 10_000_000 { + *buf++ = picohttpparser.g_digits_lut[d1] + } + if value >= 1_000_000 { + *buf++ = picohttpparser.g_digits_lut[d1 + 1] + } + if value >= 100_000 { + *buf++ = picohttpparser.g_digits_lut[d2] + } + *buf++ = picohttpparser.g_digits_lut[d2 + 1] + + *buf++ = picohttpparser.g_digits_lut[d3] + *buf++ = picohttpparser.g_digits_lut[d3 + 1] + *buf++ = picohttpparser.g_digits_lut[d4] + *buf++ = picohttpparser.g_digits_lut[d4 + 1] + } } - return unsafe { C.memcmp(dst.str, src.str, src.len) == 0 } -} -[inline] -pub fn cmpn(dst string, src string, n int) bool { - return unsafe { C.memcmp(dst.str, src.str, n) == 0 } + return unsafe { buf - buf_start } } diff --git a/vlib/picohttpparser/picohttpparser.v b/vlib/picohttpparser/picohttpparser.v index e0ad0cc1938adb..4394a17378275f 100644 --- a/vlib/picohttpparser/picohttpparser.v +++ b/vlib/picohttpparser/picohttpparser.v @@ -1,34 +1,489 @@ -// Copyright (c) 2019-2023 Alexander Medvednikov. All rights reserved. -// Use of this source code is governed by an MIT license -// that can be found in the LICENSE file. module picohttpparser -#flag -I @VEXEROOT/thirdparty/picohttpparser -#flag @VEXEROOT/thirdparty/picohttpparser/picohttpparser.o +// NOTE: picohttpparser is designed for speed. Please do some benchmarks when +// you change something in this file -#include "picohttpparser.h" +const ( + // token_char_map contains all allowed characters in HTTP headers + token_char_map = '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' + + '\0\1\0\1\1\1\1\1\0\0\1\1\0\1\1\0\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0' + + '\0\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\1\1' + + '\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\1\0\1\0' + + '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' + + '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' + + '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' + + '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' +) -struct C.phr_header { -pub: - name &char - name_len int - value &char - value_len int +fn (mut r Request) phr_parse_request_path(buf_start &u8, buf_end &u8, mut pret Pret) { + mut buf := unsafe { buf_start + 0 } + + // ADVANCE_TOKEN + method := advance_token(buf, buf_end, mut pret) + if pret.ret < 0 { + return + } + unsafe { + buf += pret.ret + } + $if trace_parse ? { + eprintln('method: ${method}') + } + // skip spaces + for { + unsafe { buf++ } + if *buf != ` ` { + break + } + } + + path := advance_token(buf, buf_end, mut pret) + if pret.ret < 0 { + return + } + $if trace_parse ? { + eprintln('path: ${path}') + } + unsafe { + buf += pret.ret + } + // skip spaces + for { + unsafe { buf++ } + if *buf != ` ` { + break + } + } + // validate + if method.len == 0 || path.len == 0 { + pret.ret = -1 + pret.err = 'error parsing request: invalid method or path' + return + } + r.method = method + r.path = path + + pret.ret = unsafe { buf - buf_start } +} + +fn (mut r Request) phr_parse_request_path_pipeline(buf_start &u8, buf_end &u8, mut pret Pret) { + mut buf := unsafe { buf_start } + method := advance_token2(buf, buf_end, mut pret) + if pret.ret < 0 { + return + } + unsafe { + buf += pret.ret + } + path := advance_token2(buf, buf_end, mut pret) + if pret.ret < 0 { + return + } + unsafe { + buf += pret.ret + } + // validate + if method.len == 0 || path.len == 0 { + pret.ret = -1 + pret.err = 'error parsing request: invalid method or path' + return + } + r.method = method + r.path = path + + for buf < buf_end { + unsafe { buf++ } + // check if following 4 characters are '\r\n\r\n' indicating a new request line + if unsafe { *(&u32(buf)) == 0x0a0d0a0d } { + unsafe { + buf += 4 + } + pret.ret = unsafe { buf - buf_start } + return + } + } + + pret.ret = -1 + pret.err = 'error parsing request: no request found' +} + +fn (mut r Request) phr_parse_request(buf_start &u8, buf_end &u8, mut pret Pret) &u8 { + // make copy of `buf_start` that can be mutated + mut buf := unsafe { buf_start } + + // skip first empty line (some clients add CRLF after POST content) + // CHECK_EOF + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + if *buf == `\r` { + unsafe { buf++ } + // EXPECT_CHAR + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + if *buf != `\n` { + pret.ret = -1 + pret.err = 'error parsing request: expected "\n" after "\r"' + return unsafe { nil } + } + } + + // parse request line + r.phr_parse_request_path(buf, buf_end, mut pret) + if pret.ret < 0 { + return unsafe { nil } + } + unsafe { + buf += pret.ret + } + minor_version := parse_http_version(buf, buf_end, mut pret) + if pret.ret < 0 { + return unsafe { nil } + } + $if trace_parse ? { + eprintln('minor_version: ${minor_version}') + } + unsafe { + buf += pret.ret + } + // CHECK_EOF + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + if *buf == `\r` { + unsafe { buf++ } + // EXPECT_CHAR + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + if *buf != `\n` { + pret.ret = -1 + pret.err = 'error parsing request: expected "\n" after "\r"' + return unsafe { nil } + } + unsafe { buf++ } + } else if *buf == `\n` { + unsafe { buf++ } + } else { + pret.ret = -1 + pret.err = 'error parsing request: expecting "\r\n" after HTTP version' + return unsafe { nil } + } + + return r.parse_headers(buf, buf_end, mut pret) } -type PPchar = &&char +[direct_array_access] +fn (mut r Request) parse_headers(buf_start &u8, buf_end &u8, mut pret Pret) &u8 { + mut buf := unsafe { buf_start } + + mut i := 0 + + for i = r.num_headers; i < max_headers; i++ { + // CHECK_EOF + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + if *buf == `\r` { + unsafe { buf++ } + // EXPECT_CHAR + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + if *buf != `\n` { + pret.ret = -1 + pret.err = 'error parsing request: expected "\n" after "\r"' + return unsafe { nil } + } + unsafe { buf++ } + + break + } else if *buf == `\n` { + unsafe { buf++ } + break + } + + if !(*buf == ` ` || *buf == `\t`) { + name_start := buf + // parsing name, but do not discard SP before colon, see + // http://www.mozilla.org/security/announce/2006/mfsa2006-33.html + for *buf != `:` { + // check if the current character is allowed in an HTTP header + if picohttpparser.token_char_map[*buf] == 0 { + $if trace_parse ? { + eprintln('invalid character! ${*buf}') + } + pret.ret = -1 + pret.err = 'error parsing request: invalid character in header "${*buf}"' + return unsafe { nil } + } + unsafe { buf++ } + + // CHECK_EOF + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + } -struct C.phr_header_t {} + name_len := unsafe { buf - name_start } + if name_len == 0 { + pret.ret = -1 + pret.err = 'error parsing request: invalid header name' + return unsafe { nil } + } + r.headers[i].name = unsafe { tos(name_start, name_len) } -fn C.phr_parse_request(buf &char, len usize, method PPchar, method_len &usize, path PPchar, path_len &usize, minor_version &int, headers &C.phr_header, num_headers &usize, last_len usize) int + unsafe { buf++ } + for { // CHECK_EOF + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + if !(*buf == ` ` || *buf == `\t`) { + break + } + unsafe { buf++ } + } + } else { + r.headers[i].name = '' + } -fn C.phr_parse_response(buf &char, len usize, minor_version &int, status &int, msg PPchar, msg_len &usize, headers &C.phr_header, num_headers &usize, last_len usize) int + mut value_len := get_token_length_to_eol(buf, buf_end, mut pret) + if pret.ret < 0 { + return unsafe { nil } + } -fn C.phr_parse_headers(buf &char, len usize, headers &C.phr_header, num_headers &usize, last_len usize) int + // TODO: strip characters + value_end := unsafe { buf + value_len } + for value_end != buf { + c := unsafe { *(value_end - 1) } + if !(c == ` ` || c == `\t`) { + break + } + unsafe { value_end-- } + } + + r.headers[i].value = unsafe { tos(buf, value_end - buf) } + r.num_headers++ + + unsafe { + buf += pret.ret + } + } + + if i == max_headers { + // too many headers + eprintln('Too many headers!') + pret.ret = -1 + pret.err = 'error parsing request: too many headers!' + return unsafe { nil } + } + + pret.ret = unsafe { buf - buf_start } + return buf +} -fn C.phr_parse_request_path(buf_start &char, len usize, method PPchar, method_len &usize, path PPchar, path_len &usize) int -fn C.phr_parse_request_path_pipeline(buf_start &char, len usize, method PPchar, method_len &usize, path PPchar, path_len &usize) int -fn C.get_date() &char +// is_complete checks if an http request is done +fn is_complete(buf_start &u8, buf_end &u8, last_len int, mut pret Pret) &u8 { + mut ret_cnt := 0 + // get the last 3 characters of the request buffer + buf := if last_len < 3 { buf_start } else { unsafe { buf_start + last_len - 3 } } -// static inline int u64toa(char* buf, uint64_t value) { -fn C.u64toa(buffer &char, value u64) int + for { + // CHECK_EOF + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + // We expect a line of an http request to end with '\r\n' + if *buf == `\r` { + unsafe { buf++ } + // CHECK_EOF + if buf == buf_end { + pret.ret = -2 + return unsafe { nil } + } + // EXPECT_CHAR_NO_CHECK + if *buf != `\n` { + // no '\n' after '\r' indicates a parse error + pret.ret = -1 + pret.err = 'error parsing request: expected "\n" after "\r"' + return unsafe { nil } + } + unsafe { buf++ } + + ret_cnt++ + } else if *buf == `\n` { + unsafe { buf++ } + ret_cnt++ + } else { + // other character + unsafe { buf++ } + ret_cnt = 0 + } + if ret_cnt == 2 { + return buf + } + } + + pret.ret = -2 + return unsafe { nil } +} + +fn parse_http_version(buf_start &u8, buf_end &u8, mut pret Pret) int { + // we want at least [HTTP/1.] to try to parse + if unsafe { buf_end - buf_start } < 9 { + pret.ret = -2 + return 0 + } + if unsafe { tos(buf_start, 7) != 'HTTP/1.' } { + pret.ret = -1 + pret.err = 'error parsing request: picohttpparser only supports HTTP/1.x' + return 0 + } + + // PARSE_INT + c := unsafe { *(buf_start + 7) } + if c < `0` || c > `9` { + pret.ret = -1 + pret.err = 'error parsing request: invalid HTTP version' + return 0 + } + pret.ret = 8 + return int(c - `0`) +} + +fn get_token_length_to_eol(buf_start &u8, buf_end &u8, mut pret Pret) int { + mut buf := unsafe { buf_start } + mut token_len := 0 + + // find non-printable char within the next 8 bytes + // HOT code: (TODO: should be manually inlined) + for _likely_(unsafe { buf_end - buf >= 8 }) { + for _ in 0 .. 8 { + if _unlikely_(!is_printable_ascii(*buf)) { + // non printable + unsafe { + goto non_printable + } + } + unsafe { buf++ } + continue + + non_printable: + // allow space and horizontal tab + if _likely_(*buf < ` ` && *buf != 9) || _unlikely_(*buf == 127) { + // found clear the line (CTL) + unsafe { + goto found_ctl + } + } + unsafe { buf++ } + } + } + // remaining characters + for { + // CHECK_EOF + if buf == buf_end { + pret.ret = -2 + return 0 + } + if _likely_(*buf < ` ` && *buf != 9) || _unlikely_(*buf == 127) { + // found clear the line (CTL) + unsafe { + goto found_ctl + } + } + unsafe { buf++ } + } + + found_ctl: + if _likely_(*buf == `\r`) { + unsafe { buf++ } + // EXPECT_CHAR + if buf == buf_end { + pret.ret = -2 + return 0 + } + if *buf != `\n` { + // no '\n' after '\r' indicates a parse error + pret.ret = -1 + pret.err = 'error parsing request: expected "\n" after "\r"' + return 0 + } + unsafe { buf++ } + token_len = unsafe { buf - 2 - buf_start } + } else if *buf == `\n` { + token_len = unsafe { buf - buf_start } + unsafe { buf++ } + } else { + pret.ret = -1 + pret.err = 'error parsing request: expecting "\r\n" after header' + return 0 + } + + if token_len == 0 { + pret.ret = 0 + return 0 + } + + pret.ret = unsafe { buf - buf_start } + return token_len +} + +// following functions are #define in the C version, but inline here for better readability + +[inline] +fn advance_token(tok_start &u8, tok_end &u8, mut pret Pret) string { + mut buf := unsafe { tok_start } + for *buf != ` ` { + if _unlikely_(!is_printable_ascii(*buf)) { + if *buf < ` ` || *buf == 127 { + pret.ret = -1 + pret.err = 'error parsing request: invalid character "${*buf}"' + return '' + } + } + unsafe { buf++ } + // CHECK_EOF + if buf == tok_end { + pret.ret = -2 + return '' + } + } + + pret.ret = unsafe { buf - tok_start } + return unsafe { tos(tok_start, pret.ret) } +} + +// advance_token2 is a less safe version of advance_token +[inline] +fn advance_token2(tok_start &u8, tok_end &u8, mut pret Pret) string { + mut len := 0 + mut i := 0 + for { + if unsafe { *(tok_start + i) == ` ` } { + len = i + for unsafe { *(tok_start + i) == ` ` } { + i++ + } + break + } + i++ + } + pret.ret = i + return unsafe { tos(tok_start, len) } +} + +[inline] +fn is_printable_ascii(c u8) bool { + return u32(c - 32) < 95 +} diff --git a/vlib/picohttpparser/request.v b/vlib/picohttpparser/request.v index 938d1c65f24fb4..1b17d266905794 100644 --- a/vlib/picohttpparser/request.v +++ b/vlib/picohttpparser/request.v @@ -1,66 +1,99 @@ module picohttpparser +const ( + max_headers = 100 +) + +pub struct Header { +pub mut: + name string + value string +} + pub struct Request { mut: prev_len int pub mut: method string path string - headers [100]C.phr_header - num_headers u64 + headers [max_headers]Header + num_headers int body string } +// Pret contains the nr of bytes read, a negative number indicates an error +struct Pret { +pub mut: + err string + // -1 indicates a parse error and -2 means the request is parsed + ret int +} + +// parse_request parses a raw HTTP request and returns the number of bytes read. +// -1 indicates a parse error and -2 means the request is parsed [inline] -pub fn (mut r Request) parse_request(s string, max_headers int) int { - method_len := usize(0) - path_len := usize(0) - minor_version := 0 - num_headers := usize(max_headers) - - pret := C.phr_parse_request(&char(s.str), s.len, voidptr(&r.method.str), &method_len, - voidptr(&r.path.str), &path_len, &minor_version, &r.headers[0], &num_headers, - r.prev_len) - if pret > 0 { - unsafe { - r.method = tos(r.method.str, int(method_len)) - r.path = tos(r.path.str, int(path_len)) +pub fn (mut r Request) parse_request(s string) !int { + mut buf := s.str + buf_end := unsafe { s.str + s.len } + + mut pret := Pret{} + // if prev_len != 0, check if the request is complete + // (a fast countermeasure against slowloris) + if r.prev_len != 0 && unsafe { is_complete(buf, buf_end, r.prev_len, mut pret) == nil } { + if pret.ret == -1 { + return error(pret.err) } - r.num_headers = u64(num_headers) + return pret.ret + } + + buf = r.phr_parse_request(buf, buf_end, mut pret) + if pret.ret == -1 { + return error(pret.err) + } + + if unsafe { buf == nil } { + return pret.ret } - r.body = unsafe { (&s.str[pret]).vstring_literal_with_len(s.len - pret) } + + pret.ret = unsafe { buf - s.str } + + r.body = unsafe { (&s.str[pret.ret]).vstring_literal_with_len(s.len - pret.ret) } r.prev_len = s.len - return pret + + // return nr of bytes + return pret.ret } +// parse_request_path sets the `path` and `method` fields [inline] -pub fn (mut r Request) parse_request_path(s string) int { - method_len := usize(0) - path_len := usize(0) - - pret := C.phr_parse_request_path(&char(s.str), s.len, voidptr(&r.method.str), &method_len, - voidptr(&r.path.str), &path_len) - if pret > 0 { - unsafe { - r.method = tos(r.method.str, int(method_len)) - r.path = tos(r.path.str, int(path_len)) - } +pub fn (mut r Request) parse_request_path(s string) !int { + mut buf := s.str + buf_end := unsafe { s.str + s.len } + + mut pret := Pret{} + r.phr_parse_request_path(buf, buf_end, mut pret) + if pret.ret == -1 { + return error(pret.err) } - return pret + + return pret.ret } +// parse_request_path_pipeline can parse the `path` and `method` of HTTP/1.1 pipelines. +// Call it again to parse the next request [inline] -pub fn (mut r Request) parse_request_path_pipeline(s string) int { - method_len := usize(0) - path_len := usize(0) - - pret := C.phr_parse_request_path_pipeline(&char(s.str), s.len, voidptr(&r.method.str), - &method_len, voidptr(&r.path.str), &path_len) - if pret > 0 { - unsafe { - r.method = tos(r.method.str, int(method_len)) - r.path = tos(r.path.str, int(path_len)) - } +pub fn (mut r Request) parse_request_path_pipeline(s string) !int { + mut buf := unsafe { s.str + r.prev_len } + buf_end := unsafe { s.str + s.len } + + mut pret := Pret{} + r.phr_parse_request_path_pipeline(buf, buf_end, mut pret) + if pret.ret == -1 { + return error(pret.err) + } + + if pret.ret > 0 { + r.prev_len = pret.ret } - return pret + return pret.ret } diff --git a/vlib/picohttpparser/response.v b/vlib/picohttpparser/response.v index 0d9e0613538a4a..f6b073ac7b21ba 100644 --- a/vlib/picohttpparser/response.v +++ b/vlib/picohttpparser/response.v @@ -36,7 +36,8 @@ pub fn (mut r Response) header(k string, v string) &Response { pub fn (mut r Response) header_date() &Response { r.write_string('Date: ') unsafe { - r.buf += cpy(r.buf, r.date, 29) + C.memcpy(r.buf, r.date, 29) + r.buf += 29 } r.write_string('\r\n') return unsafe { r } @@ -78,7 +79,7 @@ pub fn (mut r Response) json() &Response { pub fn (mut r Response) body(body string) { r.write_string('Content-Length: ') unsafe { - r.buf += C.u64toa(&char(r.buf), body.len) + r.buf += u64toa(r.buf, u64(body.len)) or { panic(err) } } r.write_string('\r\n\r\n') r.write_string(body) @@ -107,7 +108,8 @@ pub fn (mut r Response) raw(response string) { [inline] pub fn (mut r Response) end() int { n := int(i64(r.buf) - i64(r.buf_start)) - if C.write(r.fd, r.buf_start, n) != n { + // use send instead of write for windows compatibility + if C.send(r.fd, r.buf_start, n, 0) != n { return -1 } return n diff --git a/vlib/time/time_nix.c.v b/vlib/time/time_nix.c.v index e752c3d68bf1d8..48cf9aa819d144 100644 --- a/vlib/time/time_nix.c.v +++ b/vlib/time/time_nix.c.v @@ -40,7 +40,7 @@ pub fn (t Time) local() Time { } // in most systems, these are __quad_t, which is an i64 -struct C.timespec { +pub struct C.timespec { mut: tv_sec i64 tv_nsec i64