Payload (413) too large when uploading large file

[ldct-polemos]

Hello,
I noticed a potential problem when uploading large files (size > set_payload_max_length parameter). The client is a browser (Chrome) and I'm executing this js:

function uploadFile() {
    let fileInput = document.getElementById("file-upload");
    if (fileInput.files.length === 0) {
        alert("Please select a file to upload.");
        return;
    }

    let formData = new FormData();
    formData.append("file", fileInput.files[0]);

    // Upload to the current folder
    let currentPath = window.location.pathname; // Example: "/public/folder1/"

    // Ensure we don't send extra slashes
    if (!currentPath.endsWith('/')) {
        console.log(`Adding / to the end of ${currentPath}`)
        currentPath += '/';
    }

    let fullUrl = currentPath;

    console.log(`uploadFile: ${fullUrl}`);

    // Set up AbortController for timeout
    const controller = new AbortController();
    const timeout = setTimeout(() => {
        controller.abort();
    }, 60000); // 10 seconds timeout

    fetch(fullUrl, {
        method: "POST",
        body: formData,
        signal: controller.signal,
        headers: {
            "Accept": "application/json" // Expecting JSON response
        }
    })
    .then(response => {
        clearTimeout(timeout); // Clear timeout if request succeeds

        // Check if the response is OK (status code 200-299)
        if (!response.ok) {
            throw new Error(`HTTP error! status: ${response.status}`);
        }

        // Parse the response as JSON
        return response.json();
    })
    .then(data => {
        console.log("Upload successful:", data);

        // Check if the message exists in the response
        if (data.message) {
            alert(data.message);
        } else {
            alert("Upload successful!");
        }

        location.reload(); // Refresh file listing after upload
    })
    .catch(error => {
        if (error.name === 'AbortError') {
            console.error("Upload failed: Request timed out");
            alert("Error: Upload timed out. Please try again.");
        } else {
            console.error("Upload failed:", error);
            if (data.message) {
                alert(data.message);
            } else {
                alert("Error uploading file!");
            }
        }
    });
}

On the server side, I registered a Post with ContentReader. It ends up doing this:

if (req.is_multipart_form_data()) {
            LOG_INFS("Detected is_multipart_form_data()");
            // Handle multipart form data
            // Extract filename from the Content-Disposition header
            std::string uploadDir = full_path;
            std::string currentFileName;
            std::ofstream ofs;
            std::string pathFile;

            content_reader(
                // This lambda is called at the beginning of each form data field
                [&](const MultipartFormData &file) {
                    // Construct the full file path
                    LOG_INFS("reader callback:" +
                             "\n - file.filename: " + file.filename +
                             "\n - name: " + file.name +
                             "\n - content_type: " + file.content_type +
                             "\n - content_len: " + to_string(file.content.length())
                             );
                    //currentFileName = extractFileName(file.content_type);

                    pathFile = fs::path(full_path) / file.filename;
                    LOG_INFS("pathFile: " + pathFile);

                    // Open file in binary append mode for partial writes
                    ofs.open(pathFile, std::ios::binary | std::ios::app);
                    if (!ofs.is_open()) {
                        LOG_ERRORS("File <" + pathFile +  "> not opened");
                        return false;
                    }

                    return true;
                },
                // This lambda is called for each chunk of data
                [&](const char *data, size_t data_length) {
                    LOG_INFS("multipart_reader callback with: " + to_string(data_length) + " bytes");
                    // Write data to file incrementally
                    ofs.write(data, data_length);
                    return true;
                }
                );

            LOG_INFS("Finished content_reader execution, pathFile size: " + to_string(pathFile.size()));

            if(pathFile.size() > 0){
                // Close file after writing all chunks
                ofs.close();

                // // Return JSON response
                // json response;
                // response["message"] = "File uploaded successfully.";
                // response["file_name"] = currentFileName;
                // res.set_content(response.dump(), "application/json");
                LOG_INFS("Finished upload of <" + pathFile + ">");
                setSimpleRestStyleResponse(res, OK_200);
                return;
            }else{
                LOG_WARNS("0 path file size, reader callback not called?");
                return;
            }
        } else {
            LOG_ERRORS("Not a multipart form data request");
            setSimpleRestStyleResponse(res, BadRequest_400, getLastErr());
            return;
        }

        LOG_INFS("Successful upload of <" + full_path + ">");
        setSimpleRestStyleResponse(res, httplib::OK_200, "Upload successful");
}

The content_reader callbacks are never called. And by looking at the sources, I see this branch is triggered, specifically in the part "StatusCode::PayloadTooLarge_413":

template <typename T>
bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,
                  Progress progress, ContentReceiverWithProgress receiver,
                  bool decompress) {
  return prepare_content_receiver(
      x, status, std::move(receiver), decompress,
      [&](const ContentReceiverWithProgress &out) {
        auto ret = true;
        auto exceed_payload_max_length = false;

        if (is_chunked_transfer_encoding(x.headers)) {
          ret = read_content_chunked(strm, x, out);
        } else if (!has_header(x.headers, "Content-Length")) {
          ret = read_content_without_length(strm, out);
        } else {
          auto is_invalid_value = false;
          auto len = get_header_value_u64(
              x.headers, "Content-Length",
              (std::numeric_limits<uint64_t>::max)(), 0, is_invalid_value);

          if (is_invalid_value) {
            ret = false;
          } else if (len > payload_max_length) {
            exceed_payload_max_length = true;
            skip_content_with_length(strm, len);
            ret = false;
          } else if (len > 0) {
            ret = read_content_with_length(strm, len, std::move(progress), out);
          }
        }

        if (!ret) {
          status = exceed_payload_max_length ? StatusCode::PayloadTooLarge_413
                                             : StatusCode::BadRequest_400;
          exceed_payload_max_length ? std::cerr << "check exceed_payload_max_length -> **PayloadTooLarge_413**\n"
                                    : std::cerr << "check exceed_payload_max_length -> BadRequest_400\n" ;
        }
        return ret;
      });
}

I tried to force the js to use chunked transfer encoding, but without any success as this is an option managed by browser. Also I tried to remove "content-length" header in order to trigger one of these two paths:

        if (is_chunked_transfer_encoding(x.headers)) {
          ret = read_content_chunked(strm, x, out);
        } else if (!has_header(x.headers, "Content-Length")) {
          ret = read_content_without_length(strm, out);
        } else {

But again, no luck. Then I tried with curl:

echo -e "\nUploading large file:"
curl -X POST http://$DESTINATION:9000/ui/webroot \
    -H "Content-Type: application/octet-stream" \
    -H "Transfer-Encoding: chunked" \
    -H "Authorization: Bearer $TOKEN" \
    --no-buffer \
    -F "file=@config.json"

But in this case i receive a different error (not found), even if the Post cleary registers the path "/ui/webroot". Here's the logs:

read_content_chunked
read_content_with_length
res.status == -1 -> StatusCode::NotFound_404
[2025/02/20 11:18:36:575] [info] [webserver.cpp:128] - error handler
[2025/02/20 11:18:36:575] [info] [webserver.cpp:136] - Empty body in error handler, adding default rest body to response
[2025/02/20 11:18:36:575] [warn] [webserver.cpp:140] - Got error:
{
    "request": {
        "body": "--------------------------0c4d8b06ca790404\r\nContent-Disposition: attachment; name=\"file\"; filename=\"config.json\"\r\nContent-Type: application/octet-stream\r\n\r\n{\n...[CUT]...",
        "browser": false,
        "client": "127.0.0.1:32874",
        "headers": {
            "Accept": "*/*",
            "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhcHBWZXJzaW9uIjoiMS4yLjAucjE4IiwiZXhwIjoxNzQwMTMzMTE4LCJpc3MiOiJhZ2x0czJfYjA6MjU6YWE6NWY6NDg6NjAiLCJ0aW1lc3RhbXBDcmVhdGlvbiI6IjIwMjUtMDItMjAgMTA6MTg6MzYiLCJ0aW1lc3RhbXBFeHBpcmF0aW9uIjoiMjAyNS0wMi0yMSAxMDoxODozOCIsInVzck5hbWUiOiJEZXZlbG9wZXIifQ.mEicIUkyI8lE_0-yi0eBUx9TZlsKtBhQLNCvLHKsmAg",
            "Content-Type": "application/octet-stream; boundary=------------------------0c4d8b06ca790404",
            "Host": "localhost:9000",
            "LOCAL_ADDR": "127.0.0.1",
            "LOCAL_PORT": "9000",
            "REMOTE_ADDR": "127.0.0.1",
            "REMOTE_PORT": "32874",
            "Transfer-Encoding": "chunked",
            "User-Agent": "curl/7.81.0"
        },
        "method": "POST",
        "params": null,
        "path": "/ui/webroot",
        "reqNum": 12
    },
    "response": {
        "body": "<empty>",
        "headers": null,
        "status": "404 (Not Found)"
    }
}

I'm running out of ideas :( Can anybody help?
Thanks

[falbrechtskirchinger]

What exactly do you want help with? If the payload is larger than the maximum configured payload size, the request fails, as it should.
Raise the payload size or upload the file in chunks. A quick Google search turned up results on how to do the latter (example).

🤷‍♂

[ldct-polemos]

@falbrechtskirchinger thanks for you reply. I modified the js in this way:

async function uploadFile() {
    let fileInput = document.getElementById("file-upload");
    if (fileInput.files.length === 0) {
        alert("Please select a file to upload.");
        return;
    }

    const file = fileInput.files[0];
    const chunkSize = 1024 * 1024; // 1MB per chunk
    const totalChunks = Math.ceil(file.size / chunkSize);
    let currentChunk = 0;
    const fullUrl = window.location.pathname;
    const filename = encodeURIComponent(file.name);

    console.log(`Uploading ${file.name} in ${totalChunks} chunks.`);

    while (currentChunk < totalChunks) {
        const start = currentChunk * chunkSize;
        const end = Math.min(start + chunkSize, file.size);
        const chunk = file.slice(start, end);

        try {
            await uploadChunk(chunk, currentChunk, filename, totalChunks);
            console.log(`Chunk ${currentChunk + 1}/${totalChunks} uploaded successfully.`);
            currentChunk++;
        } catch (error) {
            console.error(`Failed to upload chunk ${currentChunk + 1}. Retrying...`, error);
        }
    }

    console.log('File upload completed.');
    alert('File uploaded successfully.');
    location.reload(); // Refresh file listing after upload
}

async function uploadChunk(chunk, index, filename, totalChunks, retries = 3) {
    const controller = new AbortController();
    const timeout = setTimeout(() => {
        controller.abort();
    }, 15000); // 15 seconds timeout

    const fullUrl = `${window.location.pathname}?index=${index}&filename=${filename}&totalChunks=${totalChunks}`;

    try {
        const response = await fetch(fullUrl, {
            method: "POST",
            body: chunk, // Send the chunk directly as binary data
            signal: controller.signal,
            headers: {
                "Accept": "application/json",
                "Content-Type": "application/octet-stream"
            }
        });

        clearTimeout(timeout); // Clear timeout if request succeeds

        if (!response.ok) {
            throw new Error(`HTTP error! status: ${response.status}`);
        }
    } catch (error) {
        if (retries > 0) {
            console.warn(`Retrying chunk ${index}... (${retries} retries left)`);
            await uploadChunk(chunk, index, filename, totalChunks, retries - 1);
        } else {
            console.error(`Failed to upload chunk ${index} after multiple attempts.`);
            alert(`Error: Failed to upload file. Please try again.`);
            throw error;
        }
    }
}

To send file in chunks. But again, the chunked encoding does not appear into the header:

[2025/02/20 13:22:10:375] [info] [webserver.cpp:56] - Pre-routing for new request:
{
    "body": "<empty>",
    "browser": true,
    "client": "127.0.0.1:42974",
    "headers": {
        "Accept-Encoding": "gzip, deflate, br, zstd",
        "Authorization": "Basic RGV2ZWxvcGVyOmRldmVsb3Blcg==",
        "Content-Length": "1048576",
        "Content-Type": "application/octet-stream",
        "Origin": "http://localhost:9000"
    },
    "method": "POST",
    "params": {
        "filename": "IMG_3724.MOV",
        "index": "0",
        "totalChunks": "81"
    },
    "path": "/ui/webroot",
    "reqNum": 2
}

Content length haeder is there. I see that library calls read_content_with_length() since len > 0:

          } else if (len > 0) {
            ret = read_content_with_length(strm, len, std::move(progress), out);
          }

But for some reason none of the requests (not really chunks) are routed. I end up with a lot of 404 on the client.

With other servers I worked with there was the possibility to intercept the raw request and read the data as a stream in order to make partial writes to file. With this library I'm not sure how to do it.

[falbrechtskirchinger]

Ok. Starting to see the problem. The 404s are odd. I'll dig a little in the source and let you know what I find.

Which version are you using?

With other servers I worked with there was the possibility to intercept the raw request and read the data as a stream in order to make partial writes to file. With this library I'm not sure how to do it.

#2073 should enable that as well.

[falbrechtskirchinger]

To help you debug the 404s, take a look at Server::routing() which calls Server::dispatch_request_for_content_reader with post_handlers_for_content_reader_ for req.method == "POST".

But this should all work … are you sure you don't have typo somewhere?

[ldct-polemos]

@falbrechtskirchinger I finally nailed it. The final solution is to use multiple requests for each slice of file. For my 5MB max payload I see 1MB for each slice is the maximum to avoid errors. Said this, the issue with "bad request" was that the js was passing url query parameters, but the handler expected a plain /mypath/towebroot/, without any url parameters. So I changed the js to use Form data and finally it is working. Just for reference, here's the relevant part of the implementation:

js:

...
async function uploadFile() {
    let fileInput = document.getElementById("file-upload");
    if (fileInput.files.length === 0) {
        alert("Please select a file to upload.");
        return;
    }

    const file = fileInput.files[0];
    const chunkSize = 1 * 1024 * 1024; // 1 is the highest that don't create issues on the server
    const totalChunks = Math.ceil(file.size / chunkSize);
    let currentChunk = 0;
    const filename = encodeURIComponent(file.name);

    console.log(`Uploading ${file.name} in ${totalChunks} chunks.`);

    // Reset progress bar
    const progressBar = document.getElementById('upload-progress');
    const progressPercentage = document.getElementById('upload-percentage');
    progressBar.value = 0;
    progressPercentage.textContent = '0%';

    while (currentChunk < totalChunks) {
        const start = currentChunk * chunkSize;
        const end = Math.min(start + chunkSize, file.size);
        const chunk = file.slice(start, end);

        try {
            await uploadChunk(chunk, currentChunk, filename, totalChunks);

            // Calculate and update progress
            const uploadedSize = end;
            const progress = Math.round((uploadedSize / file.size) * 100);
            progressBar.value = progress;
            progressPercentage.textContent = `${progress}%`;

            console.log(`Chunk ${currentChunk + 1}/${totalChunks} uploaded successfully.`);
            currentChunk++;
        } catch (error) {
            console.error(`Failed to upload chunk ${currentChunk + 1}.`, error);

            // Stop the loop on error
            // alert(`Error: Failed to upload chunk ${currentChunk + 1}. Please try again.`);
            break;
        }
    }

    // Final check to complete the progress bar
    if (currentChunk === totalChunks) {
        progressBar.value = 100;
        progressPercentage.textContent = '100%';
        console.log('File upload completed.');
        alert('File uploaded successfully.');
    }else{
        location.reload(); // Refresh file listing after upload
    }
}

async function uploadChunk(chunk, index, filename, totalChunks, retries = 0) {
    const controller = new AbortController();
    const timeout = setTimeout(() => {
        controller.abort();
    }, 15000); // 15 seconds timeout

    // Upload to the current folder
    let currentPath = window.location.pathname;

    // Ensure we don't send extra slashes
    if (!currentPath.endsWith('/')) {
        console.log(`Adding / to the end of ${currentPath}`);
        currentPath += '/';
    }

    let fullUrl = currentPath;
    console.log(`uploadFile: ${fullUrl}`);

    // Prepare FormData
    let formData = new FormData();
    formData.append("chunk", chunk);           // The actual file chunk
    formData.append("index", index);            // The chunk index
    formData.append("filename", filename);      // The original filename
    formData.append("totalChunks", totalChunks); // Total number of chunks

    try {
        const response = await fetch(fullUrl, {
            method: "POST",
            body: formData,
            signal: controller.signal,
            headers: {
                "Accept": "application/json"
            }
        });

        clearTimeout(timeout); // Clear timeout if request succeeds

        if (!response.ok) {
            // Attempt to parse the response as JSON
            try {
                const errorData = await response.json();
                if (errorData && errorData.message) {
                    // Throw a detailed error if message is present
                    throw new Error(`HTTP error! status: ${response.status}\nMessage: ${errorData.message}`);
                } else {
                    // Throw a generic error if no message is present
                    throw new Error(`HTTP error! status: ${response.status}`);
                }
            } catch (parseError) {
                // If parsing fails, fall back to a generic error
                throw new Error(`HTTP error! status: ${response.status}`);
            }
        }
    } catch (error) {
        if (retries > 0) {
            console.warn(`Retrying chunk ${index}... (${retries} retries left)`);
            await uploadChunk(chunk, index, filename, totalChunks, retries - 1);
        } else {
            console.error(`Failed to upload chunk ${index} after multiple attempts.`);

            // Report the error message in the alert
            let errorMessage = error.message ? error.message : "Unknown error";
            alert(`Error: Failed to upload file.\n\nDetails: ${errorMessage}\n\nPlease try again.`);

            throw error; // Re-throw to stop the upload process
        }
    }
}
...

Server side:

...
    if(req.method == "POST"){
        LOG_INFS("asyncReqHandler - POST");
        // Check if this is a rename request
        auto action_it = req.params.find("action");
        if (action_it != req.params.end() && action_it->second == "rename") {
            LOG_INFS("Rename request");
            auto new_name_it = req.params.find("new_name");
            LOG_INFS("newname: " + new_name_it->second);

            if (new_name_it == req.params.end() || new_name_it->second.empty()) {
                // res.status = BadRequest_400;
                // res.set_content("Missing new name parameter", "text/plain");
                LOG_ERRORS("Missing <newname> parameter");
                setSimpleRestStyleResponse(res, BadRequest_400, getLastErr());
                return;
            }

            string new_name = new_name_it->second;
            string new_path = fs::path(full_path).parent_path().string() + "/" + new_name;

            if (fs::exists(full_path)) {
                fs::rename(full_path, new_path);
                LOG_INFS("Successful rename of <" + full_path + "> to <" + new_path + ">");
                setSimpleRestStyleResponse(res, httplib::OK_200, "Rename successfully");
            } else {
                LOG_ERRORS("Path: <" + full_path + "> not found");
                setSimpleRestStyleResponse(res, NotFound_404, getLastErr());
            }
            return;
        }

        // Handle file upload with chunked FormData
        LOG_INFS("This is a file upload");

        if (!fs::is_directory(full_path)) {
            LOG_ERRORS("Path: <" + full_path + "> is not a directory, invalid path to directory");
            setSimpleRestStyleResponse(res, BadRequest_400, getLastErr());
            return;
        }

        if (!req.is_multipart_form_data()) {
            LOG_ERRORS("Request is not multipart/form-data");
            setSimpleRestStyleResponse(res, BadRequest_400, "Invalid form data");
            return;
        }

        // Extract FormData fields using req.files
        std::string filename;
        std::string index;
        std::string totalChunks;
        std::string chunkData;

        LOG_INFS("Parsing form data..");
        for (const auto& item : req.files) {
            const auto& file = item.second;
            if (file.name == "filename") {
                filename = file.content;
                LOG_INFS("filename: " + filename);
            } else if (file.name == "index") {
                index = file.content;
                LOG_INFS("index: " + index);
            } else if (file.name == "totalChunks") {
                totalChunks = file.content;
                LOG_INFS("totalChunks: " + totalChunks);
            } else if (file.name == "chunk") {
                chunkData = file.content;
            }
        }

        // Validate FormData fields
        if (filename.empty() || index.empty() || totalChunks.empty() || chunkData.empty()) {
            LOG_ERRORS("Missing form fields: filename, index, totalChunks, or chunk");
            setSimpleRestStyleResponse(res, BadRequest_400, "Missing form fields");
            return;
        }

        // Construct file path
        std::string filePath = full_path + "/" + filename;

        if(fs::exists(filePath) && std::stoi(index) == 0){
            LOG_ERRORS("File <" + filePath + "> already exists, rename or delete it first");
            setSimpleRestStyleResponse(res, httplib::BadRequest_400, getLastErr());
            return;
        }

        // Write chunk to file (append mode)
        std::ofstream ofs(filePath, std::ios::binary | std::ios::app);
        if (!ofs.is_open()) {
            LOG_ERRORS("Failed to open file: " + filePath);
            setSimpleRestStyleResponse(res, httplib::InternalServerError_500, "Failed to open file for writing");
            return;
        }

        ofs.write(chunkData.c_str(), chunkData.size());
        ofs.close();

        LOG_INFS("Chunk " + index + " of " + totalChunks + " received for file " + filename);

        // Check if this is the last chunk
        if (std::stoi(index) + 1 == std::stoi(totalChunks)) {
            LOG_INFS("File upload completed: " + filePath);
            setSimpleRestStyleResponse(res, httplib::OK_200, "Upload successful");
        } else {
            setSimpleRestStyleResponse(res, httplib::OK_200, "Chunk received");
        }

    }
...

Thanks for the inspiration!