From 56385409ef38d1a473af7118d126de76a9adcb0e Mon Sep 17 00:00:00 2001 From: Paolo Insogna Date: Fri, 26 Aug 2022 19:25:25 +0200 Subject: [PATCH 1/2] feat: Added on_reset callback when starting a new message. --- src/llhttp/constants.ts | 1 + src/llhttp/http.ts | 27 ++++++++++++++--- src/native/api.c | 7 +++++ src/native/api.h | 1 + test/fixtures/extra.c | 13 ++++++++ test/md-test.ts | 2 ++ test/request/connection.md | 4 +++ test/request/lenient-headers.md | 1 + test/request/pipelining.md | 54 +++++++++++++++++++++++++++++++++ test/response/connection.md | 4 +++ test/response/pipelining.md | 54 +++++++++++++++++++++++++++++++++ 11 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 test/request/pipelining.md create mode 100644 test/response/pipelining.md diff --git a/src/llhttp/constants.ts b/src/llhttp/constants.ts index 7bb7743b..d0252a57 100644 --- a/src/llhttp/constants.ts +++ b/src/llhttp/constants.ts @@ -40,6 +40,7 @@ export enum ERROR { CB_STATUS_COMPLETE = 27, CB_HEADER_FIELD_COMPLETE = 28, CB_HEADER_VALUE_COMPLETE = 29, + CB_RESET = 31, } export enum TYPE { diff --git a/src/llhttp/http.ts b/src/llhttp/http.ts index 5361a0d8..5ca36811 100644 --- a/src/llhttp/http.ts +++ b/src/llhttp/http.ts @@ -20,6 +20,7 @@ type MaybeNode = string | Match | Node; const NODES: ReadonlyArray = [ 'start', + 'after_start', 'start_req', 'start_res', 'start_req_or_res', @@ -120,6 +121,7 @@ interface ICallbackMap { readonly onStatusComplete: source.code.Code; readonly onHeaderFieldComplete: source.code.Code; readonly onHeaderValueComplete: source.code.Code; + readonly onReset: source.code.Code; } interface IMulTargets { @@ -175,6 +177,7 @@ export class HTTP { onMessageComplete: p.code.match('llhttp__on_message_complete'), onChunkHeader: p.code.match('llhttp__on_chunk_header'), onChunkComplete: p.code.match('llhttp__on_chunk_complete'), + onReset: p.code.match('llhttp__on_reset'), // Internal callbacks `src/http.c` beforeHeadersComplete: @@ -203,6 +206,7 @@ export class HTTP { p.property('i8', 'finish'); p.property('i16', 'flags'); p.property('i16', 'status_code'); + p.property('i8', 'initial_message_completed'); // Verify defaults assert.strictEqual(FINISH.SAFE, 0); @@ -233,9 +237,19 @@ export class HTTP { n('start') .match([ '\r', '\n' ], n('start')) - .otherwise(this.update('finish', FINISH.UNSAFE, - this.invokePausable('on_message_begin', - ERROR.CB_MESSAGE_BEGIN, switchType))); + .otherwise( + this.load('initial_message_completed', { + 1: this.invokePausable('on_reset', ERROR.CB_RESET, n('after_start')), + }, n('after_start')), + ); + + n('after_start').otherwise( + this.update( + 'finish', + FINISH.UNSAFE, + this.invokePausable('on_message_begin', ERROR.CB_MESSAGE_BEGIN, switchType), + ), + ); n('start_req_or_res') .peek('H', n('req_or_res_method')) @@ -890,7 +904,9 @@ export class HTTP { } n('restart') - .otherwise(this.update('finish', FINISH.SAFE, n('start'))); + .otherwise( + this.update('initial_message_completed', 1, this.update('finish', FINISH.SAFE, n('start')), + )); } private node(name: string | T): T { @@ -1042,6 +1058,9 @@ export class HTTP { case 'on_chunk_complete': cb = this.callback.onChunkComplete; break; + case 'on_reset': + cb = this.callback.onReset; + break; default: throw new Error('Unknown callback: ' + name); } diff --git a/src/native/api.c b/src/native/api.c index 277d3016..34789aba 100644 --- a/src/native/api.c +++ b/src/native/api.c @@ -67,6 +67,7 @@ const llhttp_settings_t wasm_settings = { wasm_on_message_complete, NULL, NULL, + NULL, }; @@ -370,6 +371,12 @@ int llhttp__on_chunk_complete(llhttp_t* s, const char* p, const char* endp) { return err; } +int llhttp__on_reset(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_reset); + return err; +} + /* Private */ diff --git a/src/native/api.h b/src/native/api.h index 8cfc1b59..4fa1b58f 100644 --- a/src/native/api.h +++ b/src/native/api.h @@ -54,6 +54,7 @@ struct llhttp_settings_s { */ llhttp_cb on_chunk_header; llhttp_cb on_chunk_complete; + llhttp_cb on_reset; }; /* Initialize the parser with specific type and user settings. diff --git a/test/fixtures/extra.c b/test/fixtures/extra.c index 4ee7b78a..6d988450 100644 --- a/test/fixtures/extra.c +++ b/test/fixtures/extra.c @@ -275,4 +275,17 @@ int llhttp__on_chunk_complete(llparse_t* s, const char* p, const char* endp) { #endif } +int llhttp__on_reset(llparse_t* s, const char* p, const char* endp) { + if (llparse__in_bench) + return 0; + + llparse__print(p, endp, "reset"); + + #ifdef LLHTTP__TEST_PAUSE_ON_RESET + return LLPARSE__ERROR_PAUSE; + #else + return 0; + #endif +} + #endif /* LLHTTP__TEST_HTTP */ diff --git a/test/md-test.ts b/test/md-test.ts index 9161d89d..c8c556f6 100644 --- a/test/md-test.ts +++ b/test/md-test.ts @@ -297,6 +297,7 @@ run('request/transfer-encoding'); run('request/invalid'); run('request/finish'); run('request/pausing'); +run('request/pipelining'); run('response/sample'); run('response/connection'); @@ -306,5 +307,6 @@ run('response/invalid'); run('response/finish'); run('request/lenient-version'); run('response/pausing'); +run('response/pipelining'); run('url'); diff --git a/test/request/connection.md b/test/request/connection.md index 95d6f3d0..49a185ac 100644 --- a/test/request/connection.md +++ b/test/request/connection.md @@ -48,6 +48,7 @@ off=31 len=10 span[header_value]="keep-alive" off=43 header_value complete off=45 headers complete method=4 v=1/1 flags=1 content_length=0 off=45 message complete +off=45 reset off=45 message begin off=49 len=4 span[url]="/url" off=54 url complete @@ -105,6 +106,7 @@ off=35 len=1 span[header_value]="0" off=38 header_value complete off=40 headers complete method=4 v=1/0 flags=20 content_length=0 off=40 message complete +off=40 reset off=40 message begin off=44 len=4 span[url]="/url" off=49 url complete @@ -149,6 +151,7 @@ off=108 header_value complete off=110 headers complete method=3 v=1/1 flags=20 content_length=4 off=110 len=4 span[body]="q=42" off=114 message complete +off=118 reset off=118 message begin off=122 len=1 span[url]="/" off=124 url complete @@ -283,6 +286,7 @@ off=127 header_value complete off=129 headers complete method=3 v=1/1 flags=22 content_length=4 off=129 len=4 span[body]="q=42" off=133 message complete +off=137 reset off=137 message begin off=141 len=1 span[url]="/" off=143 url complete diff --git a/test/request/lenient-headers.md b/test/request/lenient-headers.md index 8918e604..eb0de421 100644 --- a/test/request/lenient-headers.md +++ b/test/request/lenient-headers.md @@ -49,6 +49,7 @@ off=28 len=4 span[header_value]="Okay" off=34 header_value complete off=36 headers complete method=1 v=1/1 flags=0 content_length=0 off=36 message complete +off=38 reset off=38 message begin off=42 len=4 span[url]="/url" off=47 url complete diff --git a/test/request/pipelining.md b/test/request/pipelining.md new file mode 100644 index 00000000..94fad05f --- /dev/null +++ b/test/request/pipelining.md @@ -0,0 +1,54 @@ +Pipelining +========== + +## Should parse multiple events + + +```http +POST /aaa HTTP/1.1 +Content-Length: 3 + +AAA +PUT /bbb HTTP/1.1 +Content-Length: 4 + +BBBB +PATCH /ccc HTTP/1.1 +Content-Length: 5 + +CCCC +``` + +```log +off=0 message begin +off=5 len=4 span[url]="/aaa" +off=10 url complete +off=20 len=14 span[header_field]="Content-Length" +off=35 header_field complete +off=36 len=1 span[header_value]="3" +off=39 header_value complete +off=41 headers complete method=3 v=1/1 flags=20 content_length=3 +off=41 len=3 span[body]="AAA" +off=44 message complete +off=46 reset +off=46 message begin +off=50 len=4 span[url]="/bbb" +off=55 url complete +off=65 len=14 span[header_field]="Content-Length" +off=80 header_field complete +off=81 len=1 span[header_value]="4" +off=84 header_value complete +off=86 headers complete method=4 v=1/1 flags=20 content_length=4 +off=86 len=4 span[body]="BBBB" +off=90 message complete +off=92 reset +off=92 message begin +off=98 len=4 span[url]="/ccc" +off=103 url complete +off=113 len=14 span[header_field]="Content-Length" +off=128 header_field complete +off=129 len=1 span[header_value]="5" +off=132 header_value complete +off=134 headers complete method=28 v=1/1 flags=20 content_length=5 +off=134 len=4 span[body]="CCCC" +``` \ No newline at end of file diff --git a/test/response/connection.md b/test/response/connection.md index 1fa1f470..2d2552fb 100644 --- a/test/response/connection.md +++ b/test/response/connection.md @@ -86,6 +86,7 @@ off=37 len=10 span[header_value]="keep-alive" off=49 header_value complete off=51 headers complete status=204 v=1/0 flags=1 content_length=0 off=51 message complete +off=51 reset off=51 message begin off=64 len=2 span[status]="OK" ``` @@ -127,6 +128,7 @@ off=13 len=10 span[status]="No content" off=25 status complete off=27 headers complete status=204 v=1/1 flags=0 content_length=0 off=27 message complete +off=27 reset off=27 message begin off=40 len=2 span[status]="OK" ``` @@ -243,6 +245,7 @@ off=63 header_value complete off=65 headers complete status=200 v=1/1 flags=22 content_length=5 off=65 len=5 span[body]="2ad73" off=70 message complete +off=70 reset off=70 message begin off=83 len=2 span[status]="OK" ``` @@ -267,6 +270,7 @@ off=37 len=5 span[header_value]="close" off=44 header_value complete off=46 headers complete status=204 v=1/1 flags=2 content_length=0 off=46 message complete +off=46 reset off=46 message begin off=59 len=2 span[status]="OK" ``` diff --git a/test/response/pipelining.md b/test/response/pipelining.md new file mode 100644 index 00000000..26b2f190 --- /dev/null +++ b/test/response/pipelining.md @@ -0,0 +1,54 @@ +Pipelining +========== + +## Should parse multiple events + + +```http +HTTP/1.1 200 OK +Content-Length: 3 + +AAA +HTTP/1.1 201 Created +Content-Length: 4 + +BBBB +HTTP/1.1 202 Accepted +Content-Length: 5 + +CCCC +``` + +```log +off=0 message begin +off=13 len=2 span[status]="OK" +off=17 status complete +off=17 len=14 span[header_field]="Content-Length" +off=32 header_field complete +off=33 len=1 span[header_value]="3" +off=36 header_value complete +off=38 headers complete status=200 v=1/1 flags=20 content_length=3 +off=38 len=3 span[body]="AAA" +off=41 message complete +off=43 reset +off=43 message begin +off=56 len=7 span[status]="Created" +off=65 status complete +off=65 len=14 span[header_field]="Content-Length" +off=80 header_field complete +off=81 len=1 span[header_value]="4" +off=84 header_value complete +off=86 headers complete status=201 v=1/1 flags=20 content_length=4 +off=86 len=4 span[body]="BBBB" +off=90 message complete +off=92 reset +off=92 message begin +off=105 len=8 span[status]="Accepted" +off=115 status complete +off=115 len=14 span[header_field]="Content-Length" +off=130 header_field complete +off=131 len=1 span[header_value]="5" +off=134 header_value complete +off=136 headers complete status=202 v=1/1 flags=20 content_length=5 +off=136 len=4 span[body]="CCCC" +``` \ No newline at end of file From d69e974cc8979ef73e79ac7c90315facdb914e03 Mon Sep 17 00:00:00 2001 From: Paolo Insogna Date: Wed, 7 Sep 2022 13:14:30 +0200 Subject: [PATCH 2/2] feat: Added documentation. --- README.md | 223 +++++++++++++++++++++++++++++++++++++++++++++++ src/native/api.h | 2 + 2 files changed, 225 insertions(+) diff --git a/README.md b/README.md index 37edfef4..709cfeca 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,229 @@ if (err == HPE_OK) { ``` For more information on API usage, please refer to [src/native/api.h](https://github.com/nodejs/llhttp/blob/main/src/native/api.h). +## API + +### llhttp_settings_t + +The settings object contains a list of callbacks that the parser will invoke. + +The following callbacks can return `0` (proceed normally), `-1` (error) or `HPE_PAUSED` (pause the parser): + +* `on_message_begin`: Invoked when a new request/response starts. +* `on_message_complete`: Invoked when a request/response has been completedly parsed. +* `on_url_complete`: Invoked after the URL has been parsed. +* `on_status_complete`: Invoked after the status code has been parsed. +* `on_header_field_complete`: Invoked after a header name has been parsed. +* `on_header_value_complete`: Invoked after a header value has been parsed. +* `on_chunk_header`: Invoked after a new chunk is started. The current chunk length is stored + `parser->content_length`. +* `on_chunk_complete`: Invoked after a new chunk is received. +* `on_reset`: Invoked after `on_message_complete` and before `on_message_begin` when a new message is received on the same parser. This is not invoked for the first message of the parser. + +The following callbacks can return `0` (proceed normally), `-1` (error) or `HPE_USER` (error from the callback): + +* `on_url`: Invoked when the URL starts. +* `on_status`: Invoked when the status starts. +* `on_header_field`: Invoked when a new header name starts. +* `on_header_value`: Invoked when a new header value starts. + +The callback `on_headers_complete`, invoked when headers are completed, can return: + +* `0`: Proceed normally. +* `1`: Assume that request/response has no body, and proceed to parsing the next message. +* `2`: Assume absence of body (as above) and make `llhttp_execute()` return `HPE_PAUSED_UPGRADE`. +* `-1`: Error +* `HPE_PAUSED`: Pause the parser. + +### `void llhttp_init(llhttp_t* parser, llhttp_type_t type, const llhttp_settings_t* settings)` + +Initialize the parser with specific type and user settings. + +### `uint8_t llhttp_get_type(llhttp_t* parser)` + +Returns the type of the parser. + +### `uint8_t llhttp_get_http_major(llhttp_t* parser)` + +Returns the major version of the HTTP protocol of the current request/response. + +### `uint8_t llhttp_get_http_minor(llhttp_t* parser)` + +Returns the minor version of the HTTP protocol of the current request/response. + +### `uint8_t llhttp_get_method(llhttp_t* parser)` + +Returns the method of the current request. + +### `int llhttp_get_status_code(llhttp_t* parser)` + +Returns the method of the current response. + +### `uint8_t llhttp_get_upgrade(llhttp_t* parser)` + +Returns `1` if request includes the `Connection: upgrade` header. + +### `void llhttp_reset(llhttp_t* parser)` + +Reset an already initialized parser back to the start state, preserving the +existing parser type, callback settings, user data, and lenient flags. + +### `void llhttp_settings_init(llhttp_settings_t* settings)` + +Initialize the settings object. + +### `llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len)` + +Parse full or partial request/response, invoking user callbacks along the way. + +If any of `llhttp_data_cb` returns errno not equal to `HPE_OK` - the parsing interrupts, +and such errno is returned from `llhttp_execute()`. If `HPE_PAUSED` was used as a errno, +the execution can be resumed with `llhttp_resume()` call. + +In a special case of CONNECT/Upgrade request/response `HPE_PAUSED_UPGRADE` is returned +after fully parsing the request/response. If the user wishes to continue parsing, +they need to invoke `llhttp_resume_after_upgrade()`. + +**if this function ever returns a non-pause type error, it will continue to return +the same error upon each successive call up until `llhttp_init()` is called.** + +### `llhttp_errno_t llhttp_finish(llhttp_t* parser)` + +This method should be called when the other side has no further bytes to +send (e.g. shutdown of readable side of the TCP connection.) + +Requests without `Content-Length` and other messages might require treating +all incoming bytes as the part of the body, up to the last byte of the +connection. + +This method will invoke `on_message_complete()` callback if the +request was terminated safely. Otherwise a error code would be returned. + + +### `int llhttp_message_needs_eof(const llhttp_t* parser)` + +Returns `1` if the incoming message is parsed until the last byte, and has to be completed by calling `llhttp_finish()` on EOF. + +### `int llhttp_should_keep_alive(const llhttp_t* parser)` + +Returns `1` if there might be any other messages following the last that was +successfully parsed. + +### `void llhttp_pause(llhttp_t* parser)` + +Make further calls of `llhttp_execute()` return `HPE_PAUSED` and set +appropriate error reason. + +**Do not call this from user callbacks! User callbacks must return +`HPE_PAUSED` if pausing is required.** + +### `void llhttp_resume(llhttp_t* parser)` + +Might be called to resume the execution after the pause in user's callback. + +See `llhttp_execute()` above for details. + +**Call this only if `llhttp_execute()` returns `HPE_PAUSED`.** + +### `void llhttp_resume_after_upgrade(llhttp_t* parser)` + +Might be called to resume the execution after the pause in user's callback. +See `llhttp_execute()` above for details. + +**Call this only if `llhttp_execute()` returns `HPE_PAUSED_UPGRADE`** + +### `llhttp_errno_t llhttp_get_errno(const llhttp_t* parser)` + +Returns the latest error. + +### `const char* llhttp_get_error_reason(const llhttp_t* parser)` + +Returns the verbal explanation of the latest returned error. + +**User callback should set error reason when returning the error. See +`llhttp_set_error_reason()` for details.** + +### `void llhttp_set_error_reason(llhttp_t* parser, const char* reason)` + +Assign verbal description to the returned error. Must be called in user +callbacks right before returning the errno. + +**`HPE_USER` error code might be useful in user callbacks.** + +### `const char* llhttp_get_error_pos(const llhttp_t* parser)` + +Returns the pointer to the last parsed byte before the returned error. The +pointer is relative to the `data` argument of `llhttp_execute()`. + +**This method might be useful for counting the number of parsed bytes.** + +### `const char* llhttp_errno_name(llhttp_errno_t err)` + +Returns textual name of error code. + +### `const char* llhttp_method_name(llhttp_method_t method)` + +Returns textual name of HTTP method. + +### `const char* llhttp_status_name(llhttp_status_t status)` + +Returns textual name of HTTP status. + +### `void llhttp_set_lenient_headers(llhttp_t* parser, int enabled)` + +Enables/disables lenient header value parsing (disabled by default). +Lenient parsing disables header value token checks, extending llhttp's +protocol support to highly non-compliant clients/server. + +No `HPE_INVALID_HEADER_TOKEN` will be raised for incorrect header values when +lenient parsing is "on". + +**USE AT YOUR OWN RISK!** + +### `void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of conflicting `Transfer-Encoding` and +`Content-Length` headers (disabled by default). + +Normally `llhttp` would error when `Transfer-Encoding` is present in +conjunction with `Content-Length`. + +This error is important to prevent HTTP request smuggling, but may be less desirable +for small number of cases involving legacy servers. + +**USE AT YOUR OWN RISK!** + +### `void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of `Connection: close` and HTTP/1.0 +requests responses. + +Normally `llhttp` would error on (in strict mode) or discard (in loose mode) +the HTTP request/response after the request/response with `Connection: close` +and `Content-Length`. + +This is important to prevent cache poisoning attacks, +but might interact badly with outdated and insecure clients. + +With this flag the extra request/response will be parsed normally. + +**USE AT YOUR OWN RISK!** + +### `void llhttp_set_lenient_transfer_encoding(llhttp_t* parser, int enabled)` + +Enables/disables lenient handling of `Transfer-Encoding` header. + +Normally `llhttp` would error when a `Transfer-Encoding` has `chunked` value +and another value after it (either in a single header or in multiple +headers whose value are internally joined using `, `). + +This is mandated by the spec to reliably determine request body size and thus +avoid request smuggling. + +With this flag the extra value will be parsed normally. + +**USE AT YOUR OWN RISK!** + ## Build Instructions Make sure you have [Node.js](https://nodejs.org/), npm and npx installed. Then under project directory run: diff --git a/src/native/api.h b/src/native/api.h index 4fa1b58f..13cee13d 100644 --- a/src/native/api.h +++ b/src/native/api.h @@ -246,6 +246,7 @@ void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled); * * **(USE AT YOUR OWN RISK)** */ +LLHTTP_EXPORT void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled); /* Enables/disables lenient handling of `Transfer-Encoding` header. @@ -259,6 +260,7 @@ void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled); * * **(USE AT YOUR OWN RISK)** */ +LLHTTP_EXPORT void llhttp_set_lenient_transfer_encoding(llhttp_t* parser, int enabled); #ifdef __cplusplus