diff --git a/examples/ChunkRequest/ChunkRequest.ino b/examples/ChunkRequest/ChunkRequest.ino new file mode 100644 index 00000000..87cacc40 --- /dev/null +++ b/examples/ChunkRequest/ChunkRequest.ino @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright 2016-2026 Hristo Gochkov, Mathieu Carbou, Emil Muratov, Mitch Bradley + +// +// - Test for chunked encoding in requests +// + +#include +#if defined(ESP32) || defined(LIBRETINY) +#include +#include +#elif defined(ESP8266) +#include +#include +#elif defined(TARGET_RP2040) || defined(TARGET_RP2350) || defined(PICO_RP2040) || defined(PICO_RP2350) +#include +#include +#endif + +#include +#include + +using namespace asyncsrv; + +// Tests: +// +// Upload a file with PUT +// curl -T myfile.txt http://192.168.4.1/ +// +// Upload a file with PUT using chunked encoding +// curl -T bigfile.txt -H 'Transfer-Encoding: chunked' http://192.168.4.1/ +// ** Note: If the file will not fit in the available space, the server +// ** does not know that in advance due to the lack of a Content-Length header. +// ** The transfer will proceed until the filesystem fills up, then the transfer +// ** will fail and the partial file will be deleted. This works correctly with +// ** recent versions (e.g. pioarduino) of the arduinoespressif32 framework, but +// ** fails with the stale 3.20017.241212+sha.dcc1105b version due to a LittleFS +// ** bug that has since been fixed. +// +// Immediately reject a chunked PUT that will not fit in available space +// curl -T bigfile.txt -H 'Transfer-Encoding: chunked' -H 'X-Expected-Entity-Length: 99999999' http://192.168.4.1/ +// ** Note: MacOS WebDAVFS supplies the X-Expected-Entity-Length header with its +// ** chunked PUTs + +// This struct is used with _tempObject to communicate between handleBody and a subsequent handleRequest +struct RequestState { + File outFile; +}; + +void handleRequest(AsyncWebServerRequest *request) { + Serial.print(request->methodToString()); + Serial.print(" "); + Serial.println(request->url()); + + if (request->method() != HTTP_PUT) { + request->send(400); // Bad Request + return; + } + + // If request->_tempObject is not null, handleBody already + // did the necessary work for a PUT operation + auto state = static_cast(request->_tempObject); + if (state) { + if (state->outFile) { + // The file was already opened and written in handleBody so + // we are done. We will handle PUT without body data below. + state->outFile.close(); + request->send(201); // Created + } + delete state; + request->_tempObject = nullptr; + return; + } + + String path = request->url(); + + if (request->method() == HTTP_PUT) { + // This PUT code executes if the body was empty, which + // can happen if the client creates a zero-length file. + // MacOS WebDAVFS does that, then later LOCKs the file + // and issues a subsequent PUT with body contents. + +#ifdef ESP32 + File file = LittleFS.open(path, "w", true); +#else + File file = LittleFS.open(path, "w"); +#endif + + if (file) { + file.close(); + request->send(201); // Created + return; + } + request->send(403); + return; + } + + request->send(404); +} + +void handleBody(AsyncWebServerRequest *request, unsigned char *data, size_t len, size_t index, size_t total) { + if (request->method() == HTTP_PUT) { + auto state = static_cast(request->_tempObject); + if (index == 0) { + // parse the url to a proper path + String path = request->url(); + + state = new RequestState{File()}; + request->_tempObject = static_cast(state); + + if (total) { +#ifdef ESP32 + size_t avail = LittleFS.totalBytes() - LittleFS.usedBytes(); +#else + FSInfo info; + LittleFS.info(info); + auto avail = info.totalBytes - info.usedBytes; +#endif + avail -= 4096; // Reserve a block for overhead + if (total > avail) { + Serial.printf("PUT %d bytes will not fit in available space (%d).\n", total, avail); + request->send(507, "text/plain", "Too large for available storage\r\n"); + return; + } + } + Serial.print("Opening "); + Serial.print(path); + Serial.println(" from handleBody"); +#ifdef ESP32 + File file = LittleFS.open(path, "w", true); +#else + File file = LittleFS.open(path, "w"); +#endif + if (!file) { + request->send(500, "text/plain", "Cannot create the file"); + return; + } + if (file.isDirectory()) { + file.close(); + Serial.println("Cannot PUT to a directory"); + request->send(403, "text/plain", "Cannot PUT to a directory"); + return; + } + // If we already returned, the File object in request->_tempObject + // is the default-contructed one. The presence of + + std::swap(state->outFile, file); + // Now request->_tempObject contains the actual file object which owns it, + // and default-constructed File() object is in file, which will + // go out of scope + } + if (state && state->outFile) { + Serial.printf("write %d at %d\n", len, index); + auto actual = state->outFile.write(data, len); + if (actual != len) { + Serial.println("WebDAV write failed. Deleting file."); + + // Replace the File object in state with a null one + File file{}; + std::swap(state->outFile, file); + file.close(); + + String path = request->url(); + LittleFS.remove(path); + request->send(507, "text/plain", "Too large for available storage\r\n"); + return; + } + } + } +} + +static AsyncWebServer server(80); + +void setup() { + Serial.begin(115200); + +#if ASYNCWEBSERVER_WIFI_SUPPORTED +#define AP_SUBNET 100 + IPAddress local_IP(192, 168, AP_SUBNET, 1); + IPAddress gateway(192, 168, AP_SUBNET, 1); + IPAddress subnet(255, 255, 255, 0); + WiFi.softAPConfig(local_IP, gateway, subnet); + + WiFi.mode(WIFI_AP); + WiFi.softAP("esp-captive"); +#endif + +#ifdef ESP32 + LittleFS.begin(true); +#else + LittleFS.begin(); +#endif + + server.onRequestBody(handleBody); + server.onNotFound(handleRequest); + + server.begin(); +} + +void loop() { + delay(100); +} diff --git a/platformio.ini b/platformio.ini index 6c294c25..f90f1638 100644 --- a/platformio.ini +++ b/platformio.ini @@ -6,6 +6,7 @@ lib_dir = . ; src_dir = examples/Auth ; src_dir = examples/CaptivePortal ; src_dir = examples/CatchAllHandler +; src_dir = examples/ChunkRequest ; src_dir = examples/ChunkResponse ; src_dir = examples/ChunkRetryResponse ; src_dir = examples/CORS diff --git a/src/ESPAsyncWebServer.h b/src/ESPAsyncWebServer.h index 80ceb303..030b6c6b 100644 --- a/src/ESPAsyncWebServer.h +++ b/src/ESPAsyncWebServer.h @@ -280,6 +280,12 @@ class AsyncWebServerRequest { size_t _itemBufferIndex; bool _itemIsFile; + size_t _chunkStartIndex; // Offset from start of the chunked data stream + size_t _chunkOffset; // Offset into the current chunk + size_t _chunkSize; // Size of the current chunk + uint8_t _chunkedParseState; + bool _parseChunkedBytes(uint8_t *data, size_t len); + void _onPoll(); void _onAck(size_t len, uint32_t time); void _onError(int8_t error); diff --git a/src/WebRequest.cpp b/src/WebRequest.cpp index 261dde12..4c0dcf44 100644 --- a/src/WebRequest.cpp +++ b/src/WebRequest.cpp @@ -29,11 +29,20 @@ enum { PARSE_REQ_FAIL = 4 }; +enum { + CHUNK_NONE = 0, // Body transfer encoding is not chunked + CHUNK_LENGTH, // Getting chunk length - HHHH[;...] CR LF + CHUNK_EXTENSION, // Getting chunk length - HHHH[;...] CR LF + CHUNK_DATA, // Handling chunk data + CHUNK_END, // Getting chunk end marker - CR LF +}; + AsyncWebServerRequest::AsyncWebServerRequest(AsyncWebServer *s, AsyncClient *c) : _client(c), _server(s), _handler(NULL), _response(NULL), _onDisconnectfn(NULL), _temp(), _parseState(PARSE_REQ_START), _version(0), _method(HTTP_ANY), _url(), _host(), _contentType(), _boundary(), _authorization(), _reqconntype(RCT_HTTP), _authMethod(AsyncAuthType::AUTH_NONE), _isMultipart(false), _isPlainPost(false), _expectingContinue(false), _contentLength(0), _parsedLength(0), _multiParseState(0), _boundaryPosition(0), _itemStartIndex(0), - _itemSize(0), _itemName(), _itemFilename(), _itemType(), _itemValue(), _itemBuffer(0), _itemBufferIndex(0), _itemIsFile(false), _tempObject(NULL) { + _itemSize(0), _itemName(), _itemFilename(), _itemType(), _itemValue(), _itemBuffer(0), _itemBufferIndex(0), _itemIsFile(false), + _chunkedParseState(CHUNK_NONE), _tempObject(NULL) { c->onError( [](void *r, AsyncClient *c, int8_t error) { (void)c; @@ -164,6 +173,14 @@ void AsyncWebServerRequest::_onData(void *buf, size_t len) { } } } else if (_parseState == PARSE_REQ_BODY) { + if (_chunkedParseState != CHUNK_NONE) { + if (_parseChunkedBytes((uint8_t *)buf, len)) { + _parseState = PARSE_REQ_END; + _runMiddlewareChain(); + _send(); + } + break; + } // A handler should be already attached at this point in _parseLine function. // If handler does nothing (_onRequest is NULL), we don't need to really parse the body. const bool needParse = _handler && !_handler->isRequestHandlerTrivial(); @@ -334,6 +351,78 @@ bool AsyncWebServerRequest::_parseReqHead() { return true; } +// Returns true when done +bool AsyncWebServerRequest::_parseChunkedBytes(uint8_t *buf, size_t len) { + for (size_t i = 0; i < len;) { + if (_chunkedParseState == CHUNK_DATA) { + // In DATA state, we pass the bytes off to handleBody as a group + + // In order to avoid allocating an extra buffer, the data + // blocks that we pass on do not necessarily correspond to + // whole chunks. We just send however much we already have, + // anticipating that more will arrive later. handleBody() + // cannot assume that it receives entire chunks at once. + // That should not be a problem because we do not attach + // any semantic meaning to chunks. That might change if + // we were to support chunk extensions, but that seems + // unlikely since RFC9112 suggests that they are only + // useful for very specialized purposes. + size_t curLen = std::min(_chunkSize - _chunkOffset, len - i); + + // On the final zero-length chunk, _chunkSize - _chunkOffset + // will be zero, so we will call handleBody with a zero size, + // marking the end of the data stream. + + if (_handler) { + _handler->handleBody(this, buf + i, curLen, _chunkStartIndex, _contentLength); + } + _chunkOffset += curLen; + _chunkStartIndex += curLen; + i += curLen; + if (_chunkOffset == _chunkSize) { + _chunkedParseState = CHUNK_END; + } + } else { + // In other states we process the bytes one by one + uint8_t data = buf[i++]; + + if (_chunkedParseState == CHUNK_LENGTH) { + // Incrementally decode a hex number + if (data >= '0' && data <= '9') { + _chunkSize = (_chunkSize * 16) + (data - '0'); + } else if (data >= 'A' && data <= 'F') { + _chunkSize = (_chunkSize * 16) + (data - 'A' + 10); + } else if (data >= 'a' && data <= 'f') { + _chunkSize = (_chunkSize * 16) + (data - 'a' + 10); + } else if (data == ';') { + _chunkedParseState = CHUNK_EXTENSION; + } else if (data == '\n') { + _chunkOffset = 0; + _chunkedParseState = CHUNK_DATA; + } + } else if (_chunkedParseState == CHUNK_EXTENSION) { + if (data == '\n') { + // A zero length chunk marks the end of the chunk stream + _chunkOffset = 0; + _chunkedParseState = CHUNK_DATA; + } + } else if (_chunkedParseState == CHUNK_END) { + if (data == '\n') { + if (_chunkSize == 0) { + // If we needed to support trailers, we would switch to + // TRAILER state, but since we have no use case for them, + // we just stop processing the body. + return true; + } + _chunkSize = 0; + _chunkedParseState = CHUNK_LENGTH; + } + } + } + } + return false; +} + bool AsyncWebServerRequest::_parseReqHeader() { AsyncWebHeader header = AsyncWebHeader::parse(_temp); if (header) { @@ -348,7 +437,10 @@ bool AsyncWebServerRequest::_parseReqHeader() { _boundary.replace(String('"'), String()); _isMultipart = true; } - } else if (name.equalsIgnoreCase(T_Content_Length)) { + } else if (name.equalsIgnoreCase(T_Content_Length) || name.equalsIgnoreCase(T_X_Expected_Entity_Length)) { + // MacOS WebDAVFS uses X-Expected-Entity-Length to indicate the + // total length of a chunked request body. It is useful to + // determine if a PUT can possibly fit in the available space. _contentLength = atoi(value.c_str()); } else if (name.equalsIgnoreCase(T_EXPECT) && value.equalsIgnoreCase(T_100_CONTINUE)) { _expectingContinue = true; @@ -385,6 +477,17 @@ bool AsyncWebServerRequest::_parseReqHeader() { // WebEvent request can be uniquely identified by header: [Accept: text/event-stream] _reqconntype = RCT_EVENT; } + } else if (name.equalsIgnoreCase(T_Transfer_Encoding)) { + String lowcase(value); + lowcase.toLowerCase(); + + if (lowcase.indexOf("chunked") != -1) { + _chunkSize = 0; + _chunkStartIndex = 0; + _chunkedParseState = CHUNK_LENGTH; + _itemIsFile = true; + _itemFilename = _url; + } } _headers.emplace_back(std::move(header)); } @@ -680,7 +783,7 @@ void AsyncWebServerRequest::_parseLine() { String response(T_HTTP_100_CONT); _client->write(response.c_str(), response.length()); } - if (_contentLength) { + if (_contentLength || _chunkedParseState != CHUNK_NONE) { _parseState = PARSE_REQ_BODY; } else { _parseState = PARSE_REQ_END; diff --git a/src/literals.h b/src/literals.h index 52ecb6c2..ffbbdf85 100644 --- a/src/literals.h +++ b/src/literals.h @@ -101,6 +101,7 @@ static constexpr const char T_uri[] = "uri"; static constexpr const char T_username[] = "username"; static constexpr const char T_WS[] = "websocket"; static constexpr const char T_WWW_AUTH[] = "WWW-Authenticate"; +static constexpr const char T_X_Expected_Entity_Length[] = "X-Expected-Entity-Length"; // HTTP Methods static constexpr const char T_ANY[] = "ANY"; @@ -215,6 +216,7 @@ DECLARE_STR(T_HTTP_CODE_502, "Bad Gateway"); DECLARE_STR(T_HTTP_CODE_503, "Service Unavailable"); DECLARE_STR(T_HTTP_CODE_504, "Gateway Time-out"); DECLARE_STR(T_HTTP_CODE_505, "HTTP Version Not Supported"); +DECLARE_STR(T_HTTP_CODE_507, "Insufficient storage"); DECLARE_STR(T_HTTP_CODE_ANY, "Unknown code"); static constexpr const char *T_only_once_headers[] = {