http: added chunked extensions parsing and related callbacks (#193)

Author: ShogunPanda

README.md

@@ -109,6 +109,8 @@ The following callbacks can return `0` (proceed normally), `-1` (error) or `HPE_
 * `on_header_field_complete`: Invoked after a header name has been parsed.
 * `on_header_value_complete`: Invoked after a header value has been parsed.
 * `on_chunk_header`: Invoked after a new chunk is started. The current chunk length is stored in `parser->content_length`.
+* `on_chunk_extension_name_complete`: Invoked after a chunk extension name is started.
+* `on_chunk_extension_value_complete`: Invoked after a chunk extension value is started.
 * `on_chunk_complete`: Invoked after a new chunk is received. 
 * `on_reset`: Invoked after `on_message_complete` and before `on_message_begin` when a new message 
    is received on the same parser. This is not invoked for the first message of the parser.
@@ -123,6 +125,8 @@ The following callbacks can return `0` (proceed normally), `-1` (error) or `HPE_
 * `on_version`: Invoked when another character of the version is received.
 * `on_header_field`: Invoked when another character of a header name is received.
 * `on_header_value`: Invoked when another character of a header value is received.
+* `on_chunk_extension_name`: Invoked when another character of a chunk extension name is received.
+* `on_chunk_extension_value`: Invoked when another character of a extension value is received.
 
 The callback `on_headers_complete`, invoked when headers are completed, can return:
 

src/llhttp/constants.ts

@@ -42,6 +42,8 @@ export enum ERROR {
   CB_VERSION_COMPLETE = 33,
   CB_HEADER_FIELD_COMPLETE = 28,
   CB_HEADER_VALUE_COMPLETE = 29,
+  CB_CHUNK_EXTENSION_NAME_COMPLETE = 34,
+  CB_CHUNK_EXTENSION_VALUE_COMPLETE = 35,
   CB_RESET = 31,
 }
 
@@ -430,6 +432,13 @@ for (let i = 32; i <= 255; i++) {
 export const CONNECTION_TOKEN_CHARS: CharList =
   HEADER_CHARS.filter((c: string | number) => c !== 44);
 
+export const QUOTED_STRING: CharList = [ '\t', ' ' ];
+for (let i = 0x21; i <= 0xff; i++) {
+  if (i !== 0x22 && i !== 0x5c) { // All characters in ASCII except \ and "
+    QUOTED_STRING.push(i);
+  }
+}
+
 export const MAJOR = NUM_MAP;
 export const MINOR = MAJOR;
 

src/llhttp/http.ts

@@ -11,8 +11,8 @@ import {
   HTTPMode,
   LENIENT_FLAGS,
   MAJOR, METHOD_MAP, METHODS, METHODS_HTTP, METHODS_ICE, METHODS_RTSP,
-  MINOR, NUM_MAP, SPECIAL_HEADERS, STRICT_TOKEN,
-  TOKEN, TYPE,
+  MINOR, NUM_MAP, QUOTED_STRING, SPECIAL_HEADERS,
+  STRICT_TOKEN, TOKEN, TYPE,
 } from './constants';
 import { URL } from './url';
 
@@ -88,7 +88,11 @@ const NODES: ReadonlyArray<string> = [
   'chunk_size_otherwise',
   'chunk_size_almost_done',
   'chunk_size_almost_done_lf',
-  'chunk_parameters',
+  'chunk_extensions',
+  'chunk_extension_name',
+  'chunk_extension_value',
+  'chunk_extension_quoted_value',
+  'chunk_extension_quoted_value_done',
   'chunk_data',
   'chunk_data_almost_done',
   'chunk_data_almost_done_skip',
@@ -105,29 +109,33 @@ const NODES: ReadonlyArray<string> = [
 ];
 
 interface ISpanMap {
-  readonly body: source.Span;
-  readonly headerField: source.Span;
-  readonly headerValue: source.Span;
   readonly status: source.Span;
   readonly method: source.Span;
   readonly version: source.Span;
+  readonly headerField: source.Span;
+  readonly headerValue: source.Span;
+  readonly chunkExtensionName: source.Span;
+  readonly chunkExtensionValue: source.Span;
+  readonly body: source.Span;
 }
 
 interface ICallbackMap {
-  readonly afterHeadersComplete: source.code.Code;
-  readonly afterMessageComplete: source.code.Code;
-  readonly beforeHeadersComplete: source.code.Code;
-  readonly onChunkComplete: source.code.Code;
-  readonly onChunkHeader: source.code.Code;
-  readonly onHeadersComplete: source.code.Code;
   readonly onMessageBegin: source.code.Code;
-  readonly onMessageComplete: source.code.Code;
   readonly onUrlComplete: source.code.Code;
   readonly onMethodComplete: source.code.Code;
   readonly onVersionComplete: source.code.Code;
   readonly onStatusComplete: source.code.Code;
+  readonly beforeHeadersComplete: source.code.Code;
   readonly onHeaderFieldComplete: source.code.Code;
   readonly onHeaderValueComplete: source.code.Code;
+  readonly onHeadersComplete: source.code.Code;
+  readonly afterHeadersComplete: source.code.Code;
+  readonly onChunkHeader: source.code.Code;
+  readonly onChunkExtensionName: source.code.Code;
+  readonly onChunkExtensionValue: source.code.Code;
+  readonly onChunkComplete: source.code.Code;
+  readonly onMessageComplete: source.code.Code;
+  readonly afterMessageComplete: source.code.Code;
   readonly onReset: source.code.Code;
 }
 
@@ -167,6 +175,8 @@ export class HTTP {
 
     this.span = {
       body: p.span(p.code.span('llhttp__on_body')),
+      chunkExtensionName: p.span(p.code.span('llhttp__on_chunk_extension_name')),
+      chunkExtensionValue: p.span(p.code.span('llhttp__on_chunk_extension_value')),
       headerField: p.span(p.code.span('llhttp__on_header_field')),
       headerValue: p.span(p.code.span('llhttp__on_header_value')),
       method: p.span(p.code.span('llhttp__on_method')),
@@ -187,6 +197,8 @@ export class HTTP {
       onMessageBegin: p.code.match('llhttp__on_message_begin'),
       onMessageComplete: p.code.match('llhttp__on_message_complete'),
       onChunkHeader: p.code.match('llhttp__on_chunk_header'),
+      onChunkExtensionName: p.code.match('llhttp__on_chunk_extension_name_complete'),
+      onChunkExtensionValue: p.code.match('llhttp__on_chunk_extension_value_complete'),
       onChunkComplete: p.code.match('llhttp__on_chunk_complete'),
       onReset: p.code.match('llhttp__on_reset'),
 
@@ -855,15 +867,69 @@ export class HTTP {
 
     n('chunk_size_otherwise')
       .match('\r', n('chunk_size_almost_done'))
-      .match('; ', n('chunk_parameters'))
+      .match(';', n('chunk_extensions'))
       .otherwise(p.error(ERROR.INVALID_CHUNK_SIZE,
         'Invalid character in chunk size'));
 
-    n('chunk_parameters')
+    const onChunkExtensionNameCompleted = (destination: Node) => {
+      return this.invokePausable(
+       'on_chunk_extension_name', ERROR.CB_CHUNK_EXTENSION_NAME_COMPLETE, destination);
+    };
+
+    const onChunkExtensionValueCompleted = (destination: Node) => {
+      return this.invokePausable(
+       'on_chunk_extension_value', ERROR.CB_CHUNK_EXTENSION_VALUE_COMPLETE, destination);
+    };
+
+    n('chunk_extensions')
+      .match(' ', p.error(ERROR.STRICT, 'Invalid character in chunk extensions'))
+      .match('\r', p.error(ERROR.STRICT, 'Invalid character in chunk extensions'))
+      .otherwise(this.span.chunkExtensionName.start(n('chunk_extension_name')));
+
+    n('chunk_extension_name')
+      .match(STRICT_TOKEN, n('chunk_extension_name'))
+      .peek('=', this.span.chunkExtensionName.end().skipTo(
+        this.span.chunkExtensionValue.start(
+          onChunkExtensionNameCompleted(n('chunk_extension_value')),
+        ),
+      ))
+      .peek(';', this.span.chunkExtensionName.end().skipTo(
+        onChunkExtensionNameCompleted(n('chunk_extensions')),
+      ))
+      .peek('\r', this.span.chunkExtensionName.end().skipTo(
+        onChunkExtensionNameCompleted(n('chunk_size_almost_done')),
+      ))
+      .otherwise(this.span.chunkExtensionName.end().skipTo(
+        p.error(ERROR.STRICT, 'Invalid character in chunk extensions name'),
+      ));
+
+    n('chunk_extension_value')
+      .match('"', n('chunk_extension_quoted_value'))
+      .match(STRICT_TOKEN, n('chunk_extension_value'))
+      .peek(';', this.span.chunkExtensionValue.end().skipTo(
+        onChunkExtensionValueCompleted(n('chunk_size_otherwise')),
+      ))
+      .peek('\r', this.span.chunkExtensionValue.end().skipTo(
+        onChunkExtensionValueCompleted(n('chunk_size_almost_done')),
+      ))
+      .otherwise(this.span.chunkExtensionValue.end().skipTo(
+        p.error(ERROR.STRICT, 'Invalid character in chunk extensions value'),
+      ));
+
+    n('chunk_extension_quoted_value')
+      .match(QUOTED_STRING, n('chunk_extension_quoted_value'))
+      .match('"', this.span.chunkExtensionValue.end(
+        onChunkExtensionValueCompleted(n('chunk_extension_quoted_value_done')),
+      ))
+      .otherwise(this.span.chunkExtensionValue.end().skipTo(
+        p.error(ERROR.STRICT, 'Invalid character in chunk extensions quoted value'),
+      ));
+
+    n('chunk_extension_quoted_value_done')
+      .match(';', n('chunk_extensions'))
       .match('\r', n('chunk_size_almost_done'))
-      .match(HEADER_CHARS, n('chunk_parameters'))
       .otherwise(p.error(ERROR.STRICT,
-        'Invalid character in chunk parameters'));
+        'Invalid character in chunk extensions quote value'));
 
     if (this.mode === 'strict') {
       n('chunk_size_almost_done')
@@ -1094,6 +1160,12 @@ export class HTTP {
       case 'on_chunk_header':
         cb = this.callback.onChunkHeader;
         break;
+      case 'on_chunk_extension_name':
+        cb = this.callback.onChunkExtensionName;
+        break;
+      case 'on_chunk_extension_value':
+        cb = this.callback.onChunkExtensionValue;
+        break;
       case 'on_chunk_complete':
         cb = this.callback.onChunkComplete;
         break;

src/native/api.c

@@ -64,6 +64,8 @@ const llhttp_settings_t wasm_settings = {
   NULL,
   wasm_on_header_field,
   wasm_on_header_value,
+  NULL,
+  NULL,
   wasm_on_headers_complete_wrap,
   wasm_on_body,
   wasm_on_message_complete,
@@ -76,6 +78,8 @@ const llhttp_settings_t wasm_settings = {
   NULL,
   NULL,
   NULL,
+  NULL,
+  NULL,
 };
 
 
@@ -401,12 +405,41 @@ int llhttp__on_chunk_header(llhttp_t* s, const char* p, const char* endp) {
 }
 
 
+int llhttp__on_chunk_extension_name(llhttp_t* s, const char* p, const char* endp) {
+  int err;
+  SPAN_CALLBACK_MAYBE(s, on_chunk_extension_name, p, endp - p);
+  return err;
+}
+
+
+int llhttp__on_chunk_extension_name_complete(llhttp_t* s, const char* p, const char* endp) {
+  int err;
+  CALLBACK_MAYBE(s, on_chunk_extension_name_complete);
+  return err;
+}
+
+
+int llhttp__on_chunk_extension_value(llhttp_t* s, const char* p, const char* endp) {
+  int err;
+  SPAN_CALLBACK_MAYBE(s, on_chunk_extension_value, p, endp - p);
+  return err;
+}
+
+
+int llhttp__on_chunk_extension_value_complete(llhttp_t* s, const char* p, const char* endp) {
+  int err;
+  CALLBACK_MAYBE(s, on_chunk_extension_value_complete);
+  return err;
+}
+
+
 int llhttp__on_chunk_complete(llhttp_t* s, const char* p, const char* endp) {
   int err;
   CALLBACK_MAYBE(s, on_chunk_complete);
   return err;
 }
 
+
 int llhttp__on_reset(llhttp_t* s, const char* p, const char* endp) {
   int err;
   CALLBACK_MAYBE(s, on_reset);

src/native/api.h

@@ -28,6 +28,8 @@ struct llhttp_settings_s {
   llhttp_data_cb on_version;
   llhttp_data_cb on_header_field;
   llhttp_data_cb on_header_value;
+  llhttp_data_cb      on_chunk_extension_name;
+  llhttp_data_cb      on_chunk_extension_value;
 
   /* Possible return values:
    * 0  - Proceed normally
@@ -51,6 +53,8 @@ struct llhttp_settings_s {
   llhttp_cb      on_version_complete;
   llhttp_cb      on_header_field_complete;
   llhttp_cb      on_header_value_complete;
+  llhttp_cb      on_chunk_extension_name_complete;
+  llhttp_cb      on_chunk_extension_value_complete;
 
   /* When on_chunk_header is called, the current chunk length is stored
    * in parser->content_length.

test/fixtures/extra.c

@@ -305,6 +305,50 @@ int llhttp__on_chunk_header(llparse_t* s, const char* p, const char* endp) {
 }
 
 
+int llhttp__on_chunk_extension_name(llparse_t* s, const char* p, const char* endp) {
+  if (llparse__in_bench)
+    return 0;
+
+  return llparse__print_span("chunk_extension_name", p, endp);
+}
+
+
+int llhttp__on_chunk_extension_name_complete(llparse_t* s, const char* p, const char* endp) {
+  if (llparse__in_bench)
+    return 0;
+
+  llparse__print(p, endp, "chunk_extension_name complete");
+
+  #ifdef LLHTTP__TEST_PAUSE_ON_CHUNK_EXTENSION_NAME
+    return LLPARSE__ERROR_PAUSE;
+  #else
+    return 0;
+  #endif
+}
+
+
+int llhttp__on_chunk_extension_value(llparse_t* s, const char* p, const char* endp) {
+  if (llparse__in_bench)
+    return 0;
+
+  return llparse__print_span("chunk_extension_value", p, endp);
+}
+
+
+int llhttp__on_chunk_extension_value_complete(llparse_t* s, const char* p, const char* endp) {
+  if (llparse__in_bench)
+    return 0;
+
+  llparse__print(p, endp, "chunk_extension_value complete");
+
+  #ifdef LLHTTP__TEST_PAUSE_ON_CHUNK_EXTENSION_VALUE
+    return LLPARSE__ERROR_PAUSE;
+  #else
+    return 0;
+  #endif
+}
+
+
 int llhttp__on_chunk_complete(llparse_t* s, const char* p, const char* endp) {
   if (llparse__in_bench)
     return 0;

test/md-test.ts

@@ -261,6 +261,10 @@ function run(name: string): void {
 
       for (const mode of modes) {
         for (const ty of types) {
+          if (meta.skip === true || (process.env.ONLY === 'true' && !meta.only)) {
+            continue;
+          }
+
           runSingleTest(mode, ty, meta, input, fullExpected);
         }
       }