How to go about tracking down this error? #523

espruino-discuss · 2025-02-10T18:43:46Z

espruino-discuss
Feb 10, 2025
Maintainer

Posted at 2016-01-13 by Snerkle

I am using a pico with an ESP8266 for wifi.
I'm writing code that buffers the body of in incoming http request and attempts to write it to a file.

For small files (around 1-2K) it works. As soon as I try with a larger file I get the following error, I think its while I'm still buffering the body of the request.

ERROR: Out of Memory!
at line 1 col 106
...tr(0,c),h[a]=h[a].substr(c)):(b=h[a],h[a]="");return b}retur...
                              ^
in function called from system
ERROR: Error processing Serial data handler - removing it.
WARNING: Unable to create string as not enough memory
WARNING: Unable to create string as not enough memory
Execution Interrupted during event processing.
at line 1 col 189
...+1);h[b[0]]+=a.substr(c+1,f);return"+IPD,"+b[0]+","+(b[1]-f)...
         in function called from system
at line 1 col 106
...tr(0,c),h[a]=h[a].substr(c)):(b=h[a],h[a]="");return b}retur...
                            at line 2 col 73
...E3,function(c){d[a]=void 0})
                              ^
in function called from sysExecution Interrupted

I'm at a bit of a loss of how to go about tracking this down as it looks like its somewhere in the ESP8266WIFI_0v25 or the AT library. I could be wrong, but it doesn't look like any of my code.

Any help would be appreciated.

gfwilliams · 2025-02-10T18:43:49Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-13 by @gfwilliams

Can you post up your code?

The error is in the ESP8266 library, but that's probably not at fault - there's just so little memory available that it can't allocate memory when it needs it, and it crashes.

If you want to write to a file, I'd write it a bit at a time - each time you get new data, add it to the end. You might be able to use pipe...

There's an example of transmitting a file on the Internet page:

function onPageRequest(req, res) {
  var a = url.parse(req.url, true);
  var f = E.openFile(a.pathname, "r");
  if (f !== undefined) {
    res.writeHead(200, {'Content-Type': 'text/plain'});
    f.pipe(res); // streams the file to the HTTP response
  } else {
    res.writeHead(404, {'Content-Type': 'text/plain'});
    res.end("404: Page "+a.pathname+" not found");
  }
}
require("http").createServer(onPageRequest).listen(8080);

You might just be able to reverse that to:

function onPageRequest(req, res) {
  var a = url.parse(req.url, true);
  // ...
  var f = E.openFile(a.pathname, "w");
  req.pipe(f); // it should auto-close
}
require("http").createServer(onPageRequest).listen(8080);

but I haven't tried it...

0 replies

espruino-discuss · 2025-02-10T18:43:51Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-13 by DrAzzy

Can you post the code that's doing it?

Also - you're working with an Espruino board (or some other STM32 board running Espruino) connected to an ESP8266, right? (as opposed to running it on the ESP8266 itself - which is still a little rough, and is really tight on memory)

0 replies

gfwilliams · 2025-02-10T18:43:54Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-13 by @gfwilliams

Actually I just dug this out of some personal code I had:

function onPageRequest(req, res) {
  var header = "";
  req.on('data', function(d) { 
    header += d;
    var idx = header.indexOf("\r\n\r\n");
    if (idx>=0) {
      var f = E.openFile("f.txt", "w");
      f.write(header.substring(idx+4));
      header = undefined;
      req.removeAllListeners('data');
      req.pipe(f);
    }
  });
  res.writeHead(200, {'Content-Type': 'text/html'});
  res.write('<html><body><form method="post" enctype="multipart/form-data">');
  res.write('<input type="file" name="file" id="file"><input type="submit">');
  res.end('</form></body></html>');
}
require('http').createServer(onPageRequest).listen(8080);

It works fine - the only problem is that you end up with the HTTP 'boundary' on the end of the file - so if you care about that then it probably makes sense to handle it with on('data' instead and keep checking for the boundary (which is more painful than it sounds).

0 replies

gfwilliams · 2025-02-10T18:43:57Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-13 by @gfwilliams

And I just tweaked it:

var fileNo = 0;

function onPageRequest(req, res) {
  var d = "";
  var boundary;
  var f;
  req.on('close', function() { 
    if (f) {
      f.close();
      console.log("File uploaded");
    }
  });
  req.on('data', function(data) { 
    d += data;
    if (f) {
      var idx = d.indexOf(boundary);
      /* ensure we always leave some in our buffer
      in case boundary straddles packets */
      if (idx>=0) {
        console.log("Boundary found - starting again");
        f.write(d.substring(0, idx));
        f.close();
        f = undefined;
      } else {
        idx = d.length-boundary.length; 
        f.write(d.substring(0, idx));
      }
      d = d.substring(idx);
    } else {
      var idx = d.indexOf("\r\n\r\n");
      if (idx>=0) {
        var header = d.substring(0,idx).split("\r\n");
        boundary = "\r\n"+header.shift();
        header = header.map(function(x){return x.split(": ");});
        console.log(JSON.stringify(header));  
        // choose a filename
        var fileName = "f"+fileNo+".txt";
        console.log("Writing to "+fileName);
        f = E.openFile(fileName, "w");
        fileNo++;
        d = d.substring(idx+4);
      }
    }
  });
  res.writeHead(200, {'Content-Type': 'text/html'});
  res.write('<html><body><form method="post" enctype="multipart/form-data">');
  res.write('<input type="file" name="file1" id="file"><input type="file" name="file2" id="file"><input type="submit">');
  res.end('</form></body></html>');
}
require('http').createServer(onPageRequest).listen(8080);

So now it handles boundaries correctly, and allows multiple files to be uploaded in one go.

... I guess this should be a module of some kind?

0 replies

espruino-discuss · 2025-02-10T18:43:59Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-14 by Snerkle

Thanks for all the thought provoking input. I'll try to summarise what I've learnt in the past 24 hours.

@gordon pointed me in the right direction to the ESP8266WIFI_0v25 module. I'd failed to remember that modules get minified, which explains why I couldn't identify the module myself. So thanks.
@drazzy, yes this is a Pico with the Esp8266 connected via the shim.
So I instrumented the ESP8266WIFI_0v25 module and found that the problem lies there or in the AT module: once the serial data starts to arrive from the Esp8266 the ser.on('data',... gets all the priority and all the incoming data is buffered by the ipdHandler in sockData[sckt].
only once all the data is buffered, then the socketServer calls netcallback.recv to start getting chunks to send on to req.on('data',...
this is the point where I got the out of memory error: because JavaScript does expensive string copies for functions like String.slice() and String.substr(); as the netcallback.recv returns a chunk from sockData[sckt] it tries to prune the chunk from the buffer causing it to copy the larger remaining part of the buffer

So there are plenty of places where you could run out of memory with this behaviour, basically because the code doesn't let you process and discard each incoming chunk of data as it arrives, but buffers it and then chunks it up after it is all buffered.

Ideally the ESP8266WIFI_0v25 module and/or the AT module need to be redesigned to behave as a stream, and not just to emulate a stream by actually buffering everything and then chucking out from the buffer.

I hope all this makes sense.

0 replies

gfwilliams · 2025-02-10T18:44:02Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-14 by @gfwilliams

Well, ideally the Network API would allow data to be 'pushed' into it. However I still think the problem is in the way you're using the HTTP code - we need to see your code.

So the way Espruino works is:

Data comes in via Serial, it's handled in an IRQ and buffered
Each time around the idle loop, Espruino empties that buffered data, calling AT and ESP8266WiFi modules, which put the data into sockData.
Immediately after that in the idle loop, the socket server code in Espruino calls ESP8266WiFi's recv, which then passes chunks of data back to it, where it is passed on via the socket.on('data' handler. However it often only does this 64 bytes at a time.

So if Espruino isn't busy, it keeps going round the idle loop, emptying the buffers and everything is fine.

However, if your code is taking too long to execute, Espruino doesn't go around the idle loop enough times, and data builds up in the buffers (probably sockData).

To be honest I don't see how redesigning the ESP8266 network will help in this case. It'll mean that the data that comes in just gets pushed straight to your code, where it won't be handled fast enough and Espruino will still fail with out of memory, just in a different place.

0 replies

espruino-discuss · 2025-02-10T18:44:04Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-14 by Snerkle

Ok, I'll post my code tomorrow when I get back to my computer. I'm not doing anything time consuming, I'm actually just buffering the data again for now until I solve this problem and can then work more like a pipe/stream.

I take your last point and am inclined to agree.

What I was expecting to see was each of the three bullet-pointed steps happen mostly sequentially for each incoming packet of data, so overall it's like a stream or pipe with only a little buffering to keep things moving.
However what I am observing is that the 3rd bullet-point doesn't start until all the many, available incoming data packets have been buffered in points 1&2. Then when the socket server does start it results in the out of memory condition as the larger part of the buffer then get copied because we're chunking up the buffer in JavaScript (netcallback.recv) and not C/C++ (socketServer.c)

I'll also send the output from my instrumentation so you can see this happening.

0 replies

gfwilliams · 2025-02-10T18:44:07Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-14 by @gfwilliams

the 3rd bullet-point doesn't start until all the many, available incoming data packets have been buffered

Are you using a high baud rate, or is this still at 115200 baud?

It looks to me like it starts as soon as data starts being received (so it doesn't have to have go to the end of an IPD packet). However it tries to be 'up to date' with the characters coming in from the UART before it moves on. It could be Espruino's handling of input data (rather than anything network-related) that's causing problems.

For example it currently does:

var limit = 128;
while (hasInputCharacters() && limit--) {
  var data = "";
  while (data += popInputCharactersForDevice(SERIAL1));
  handleInputCharacters(data);
}

But I imagine what could happen is if characters keep coming in, it calls javascript with something a bit like:

"lotsofdatalotsofdatalotsofdatalotsofdatalotsofdatalotsofdatalotsofdata"
"l" // by the time this is processed, another character has come in
"o" // and so on
"t"
"s"

0 replies

espruino-discuss · 2025-02-10T18:44:10Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-15 by Snerkle

Hi, I've spent many hours now setting up the tests and investigating different settings. Note: this code doesn't do anything with the incoming data, just counts the bytes.

The good news is that with 'well matched' settings the pipe/stream will flow all the way from the ipdHandler to the req.on('data',... in your own code. More on what 'well matched' means later.
The bad news is that under default settings (i.e. 115200 baud usart to ESP8266 and a 64 byte CHUNK size in socketserver.c) the netCallbacks.recv never gets the chance to get going: the ipdHandler (or more likely the ser.on("data",... in AT.js) dominates and buffers all the incoming serial data from the ESP8266 into sockData[sckt] before the netCallbacks.recv gets going. The memory pressure doubles (more or less) on the following calls to netCallbacks.recv as it tries to remove a 64 byte chunk from the beginning of sockData[sckt]

    if (sockData[sckt]) {
      /** INSTRUMENTED =>**/ var len = sockData[sckt].length;
      var r;
      if (sockData[sckt].length > maxLen) {
        r = sockData[sckt].substr(0,maxLen);
        sockData[sckt] = sockData[sckt].substr(maxLen);
      } else {

that second .substr(...) is the culprit as it makes a copy of the whole buffer (less 64 (maxlen) bytes).
On a pico running only the attached code, the maximum http message body (file upload) is just a bit less than 20K before an out-of-memory error occurs. I've attached a log generated from my instrumented copy of ESP8266WIFI_0v25 (also attached) where you can see that netCallbacks.recv only gets invoked once or twice by the time ipdHandler has buffered the whole http message.

If you play around with dropping the baud rate down to say 9600 you can see that while the serial data is arriving, netCallbacks.recv gets invoked only every ~500ms.

So what are some 'well matched' settings?
Well I'd been playing around with compiling my own build of Espruino just to play around with the CHUNK size in socketserver.c.
So as an example: a CHUNK size of 1460 matched with a baudrate of 19200 gets the stream flowing nicely (actually I can bring the baudrate upto 24000 with this chunk size) and the maximum size of the sockData[sckt] never gets much larger than the CHUNK size. This isn't exactly a fantastic baudrate, but I can stream in a file of any size (tested upto 300K, but saw that the buffer always stayed small), although I'm just counting the bytes and nothing more 'busy'.

So what next?

flow control isn't an option as the ESP8266 only does hardware flow control with AT firmware, and the USART2 on the Pico doesn't support hardware flow control
using a bigger CHUNK size and a low baudrate might be an option, memory permitting for a given project
rewriting the ESP8266WIFI_0v25 module in native code in Espruino for better memory efficiency (one and only one copy of the data until it is consumed?) and optimising the balance between receiving serial data and scheduling javaScript execution
if this isn't an option and we have to acknowledge the maximum buffer size that can be achieved with sockData then the memory pressure can be managed by using an index into the buffer for the start of the next chunk, and only removing delivered chunks when the buffer is at least half consumed(?)

Attachments:

0 replies

gfwilliams · 2025-02-10T18:44:13Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-15 by @gfwilliams

Not sure I really understand this... so you have:

//Handle +IPD input data from ESP8266
function ipdHandler(line) {
  var colon = line.indexOf(":");
  if (colon<0) return line; // not enough data here at the moment
  var parms = line.substring(5,colon).split(",");
  parms[1] = 0|parms[1];
  var len = line.length-(colon+1);
  if (len>=parms[1]) {
   // we have everything
   sockData[parms[0]] += line.substr(colon+1,parms[1]);
   /** INSTRUMENTED =>**/ console.debug('ipdHandler (empty): ', line.length, ' bytes in packet, ', sockData[parms[0]].length, ' bytes in total');
   return line.substr(colon+parms[1]+1); // return anything else
  } else { 
   // still some to get
   sockData[parms[0]] += line.substr(colon+1,len);
   ///** INSTRUMENTED =>**/ console.debug('ipdHandler (more) : ', line.length, ' bytes in packet, ', sockData[parms[0]].length, ' bytes in total');
   return "+IPD,"+parms[0]+","+(parms[1]-len)+":"; // return IPD so we get called next time
  }
}

So it should be reporting back every single time it gets more data in?

But it actually reports:

ipdHandler (empty):  61  bytes in packet,  591  bytes in total
ipdHandler (empty):  63  bytes in packet,  2051  bytes in total
ipdHandler (empty):  64  bytes in packet,  3511  bytes in total
ipdHandler (empty):  65  bytes in packet,  4971  bytes in total
ipdHandler (empty):  67  bytes in packet,  6431  bytes in total
recv [sckt:0] (deltaTime:  949377358.046 ) :  64  bytes of  7063  sent to socketServer
ipdHandler (empty):  67  bytes in packet,  7827  bytes in total
ipdHandler (empty):  69  bytes in packet,  9287  bytes in total
ipdHandler (empty):  70  bytes in packet,  10747  bytes in total
ipdHandler (empty):  72  bytes in packet,  12207  bytes in total
ipdHandler (empty):  72  bytes in packet,  13667  bytes in total
ipdHandler (empty):  73  bytes in packet,  14632  bytes in total

I'd have expected bytes in total to increment by a bit less than 'bytes in packet' each time, but it's going up by about 1500 bytes!

I think one solution would be to change the netRecv function such that it used JsVars rather than a fixed size buffer. That way it'd read as much data as the network connection was willing to give it.

0 replies

espruino-discuss · 2025-02-10T18:44:15Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-15 by Snerkle

OK I see your confusion: with the instrumentation in ipdHandler, line 16 (above) is commented out ///* so you don't see the packets reported where it is known there is more data to come i.e. ipdHandler (more) as opposed to ipdHandler (empty). This was just to avoid flooding the logging. If you want to see everything as it arrives just remove the first // on line 16 (above) and you'll get the lot.

So if I understand you right: socketserver.c would not use CHUNK to define a fixed buffer length for incoming data but just take whatever netRecv gets from netCallbacks.recv with no maxlen specified, as it will always return everything in sockData[sckt].

If I have this right, then sockData[sckt] will still be buffering around 6-8K (in your log above: 7063 bytes on the first recv) as recv only gets to run every ~500ms while serial data is arriving. Is there a way to make this more frequent, say once every ~100ms to keep the buffering down? Even a 6-8K buffer will stretch available memory.

P.S. the deltaTime in the logging is only meaningful on the second call to recv onwards.

My other crazy plan, that I will ask a few question of @tve to qualify if it is worth the effort, is to to run Espruino on the ESP8266 which implements the SDK's flow control for TCP sockets espconn_recv_hold and espconn_recv_unhold. Then write some javaScript to run on the ESP8266 paired with a module on the Pico that effectively proxies the ESP8266 Wifi module across the serial connection between the two devices, along with a kind of proxy for the netCallbacks on the Pico side paired with the net and sockets modules on the ESP8266 side. Waddayoureckon? Crazy huh! If this can be made to work, we get to use the memory on the ESP8266 for some of the buffering and we get TCP flow control.

0 replies

gfwilliams · 2025-02-10T18:44:18Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-15 by @gfwilliams

as recv only gets to run every ~500ms while serial data is arriving

I'm extremely surprised at that - if so then there's some very big, deep-seated problem.

As I'd said above, the idle loop in Espruino will only process 128 items from the input queue (4 chars in each item, so 512 chars total) before carrying on - and then it will call into the socket handling code.

So yes, it should run more often than once every 500ms, and the max stored data should be around 512 bytes.

Honestly, I think running custom firmware on the ESP8266 is overkill :) This really should be pretty simple to fix.

Of course if the ESP8266 firmware could implement flow control, that'd be pretty cool - but it really shouldn't be needed.

0 replies

espruino-discuss · 2025-02-10T18:44:21Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-15 by Snerkle

I'll produce another log for you (attached), or you can try it for yourself: if you drop the baudrate in the test down to 9600, you'll see you get more `recv` events before all the serial data has been buffered by `ipdHandler` and the `deltatime` between each one is ~500ms. Once the `idpHandler` is finished then the `deltatime` between each `recv` event is down to ~14ms

Attachments:

test2.log

0 replies

gfwilliams · 2025-02-10T18:44:24Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-15 by @gfwilliams

Just looking at this now - yes, there's something going wrong with the idle loop. While it counts events as it should, it's not counting when it can fire off a whole batch of events in a go.

As a result it limits to 128 calls on on('data' rather than 512 characters, and that's way too much.

I've just fixed that, but the limit of 64 bytes for a call to recv is still too few, so I'll make some changes to the network API too.

0 replies

gfwilliams · 2025-02-10T18:44:27Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-15 by @gfwilliams

I've pushed some code, so the latest GitHub builds will at least have the idle loop bug fixed, and have big enough buffers for HTTP/Sockets so that data gets drained quickly enough.

However, now I increased the buffer sizes, I encountered an issue where res.end() tries to close the connection, so that's broken things as well! I think the solution there is going to be to count the received bytes and then compare them against the Content-Length header - but I think that's for next week :)

0 replies

gfwilliams · 2025-02-10T19:14:32Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-19 by @gfwilliams

Ok, the HTTP server now only closes once Content-Length bytes have been received (as of the latest Git build)

0 replies

espruino-discuss · 2025-02-10T19:14:40Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-19 by Snerkle

OK, I've been doing a lot of testing in the meantime and found:

it all works nicely if you don't do anything that blocks in req.on('data',...
- so counting bytes; buffering into a string; etc... all works well
things you'd like to do like file i/o don't work so well, causing incoming serial packets from the ESP8266 to get dropped
- creating a file with E.openFile() takes a long time and blocks
- file write with File.write() can cause packet loss, but if you rebuffer into 2048 blocks its quite good, File.pipe(...,{chunkSize:2048}) can do this nicely but...
- File.pipe() never ends because of the issue you've now addressed, so I will test now with this

If I drop the baudrate of the ESP8266 usart down to 19200, I can make things work fairly reliably. But this speed isn't ideal for more than a proof-of-concept.

The other thing I see happen (sometimes) is long (2 seconds plus) pauses that I assume are garbage collection. Sometimes the streaming of data survives this, sometimes it doesn't and again incoming serial packets from the ESP8266 to get dropped

Is there a way to request the GC to pause and unpause upon request?

My write speed to my SD card is effectively ~650000 bps so the file writing shouldn't be an issue for incoming data at 115200 bps, but something is blocking the serial interrupt and a few packets always get dropped at speeds over 19200 i.e. 38400 or higher.

0 replies

gfwilliams · 2025-02-10T19:14:43Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-19 by @gfwilliams

That's odd - GC is actually extremely quick (just a few milliseconds) so I doubt that's the issue. Since Serial and USB share the same input buffer, could it be that there's just a big flood of data, and the serial buffer gets so full it can't even accept characters from USB?

Glad it's working with pipe anyway - since effectively 512 bytes get buffered, you've got ~44ms at 115200 baud in which to write the data out (in practice it's less though). How are you connecting the SD card? Is it possible that you could get a bit more out of it using hardware SPI with a higher baud rate.

Having said that, hardware SPI uses interrupts to handle the received characters, and it's possible that those interrupts end up blocking the serial interrupts and you lose data. You could try with software SPI, which might be a tad slower but wouldn't use interrupts so might mean you don't end up losing data?

0 replies

espruino-discuss · 2025-02-10T19:14:46Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-20 by Snerkle

OK, I've tried both hardware and software SPI and end up with the same problem: I've opened a new post to follow that issue separately, although I believe it is now the root cause of my problems with dropping incoming packets of serial data from the ESP8266.

Meanwhile I'll let you know that the last binary/build that I can use is e54a6b982545da111b2f24d725fb30b0279c11da , anything after that has a regression error when trying to require a module

Uncaught SyntaxError: Got ?[255] expected EOF
 at line 1 col 257
...t(),h.cmd(a[0],a[1],a[2]))}ÿsco
                              ^
in function "cmd" called from line 2 col 79
...er AT+RST");else return cb})
                              ^
in function "reset" called from line 1 col 152
...ncs.reset(connectedCallback);return wifiFuncs
                              ^
in function "connect" called from line 24 col 2
});
 ^

you'll notice that \xFF character on line 3 that isn't there, and I don't get this error with the build linked just above.

I like where its heading with detecting the end of the body by comparing the bytes received with the Content-Length header. This is what I've been doing in my javaScript in the req.on('data',... for a while now. It looks like the implementation is still unfinished, and it currently also logs the message Closing now to the console from socketserver.c. And req.pipe(...) still doesn't end as yet.

I'd like to propose a few things at this point:

1. can the (new) chunksize parameter be user-definable if using NewtworkJS type 'drivers'. The place this could go is in the 'driver' module, in my case ESP8266WIFI_0v25 by including a chunksize attribute in the object passed to require("NetworkJS").create(obj), that if present, overrides the default value of 536 specified in network_js.c. As I have shown on line 2 here:

var netCallbacks = {
  chunkSize: 1460,
  create : function(host, port) {
    /* Create a socket and return its index, host is a string, port is an integer.
    If host isn't defined, create a server socket */  
    if (host===undefined) {
.
.
.
  require("NetworkJS").create(netCallbacks);

I would try and add it myself but I still haven't got my head around handling jsVars in the Espruino source code. This would remove the trouble of having to make a new build to try out different socket buffer sizes.

2. I propose it is a good opportunity to introduce the req.on('end',... event into Espruino. This is how node works with its ReadableStream implementation. Checkout https://nodejs.org/en/docs/guides/anatomy-of-an-http-transaction/#request-body for some background.

an http transaction works on a 'half-closed' basis between client and server
the client (half-)closes the transaction after transmitting the request body with a tcp FIN packet
of course devices like the ESP8266 don't make this visible to us, so
it is typical to count the incoming bytes and compare with the Content-Length header and req.emit('end') when everything has arrived
a timeout is probably also required (reset whenever data does arrive) in case the client stops sending, or some of the data never arrives - if there is a timeout, it should req.emit('error')
the server only closes the transaction when it issues a res.end() or the socket is taken down for another reason (its been 'half-closed' until now)
when the client gets the result of the res.end() the http transaction is complete
if the client never gets a complete response (because the socket goes down for another reason) its up to the client to decide how to deal with this

The upshot of this approach is that the http server does not and should not have to issue a res.end() just to get a close (or end) event and then determine if the the whole of the request body has been received: an end event should happen as soon as the byte count is right.

I've noticed a lot of the sample/tutorial code for an http server does this 'early' res.end() to close the transaction/socket, which is fine if you are only expecting an http request header anyway, but not if you are expecting to receive/stream in an http request body. In fact when the res.end() is done you'd probably see a req.on('close',... event fire but I'll have to check this out in node to be sure.

Gosh this post got long but I really hope to make this work on the Espruino so I can complete my larger project, which by the way is an attempt to implement a light-weight, modular express (V4) like http router for the Espruino to simplify building http server apps along with my first http server app which would be a web based UI to manage content on an attached SD storage card, including drag-and-drop file uploads - so you can see why reliably receiving long http request bodies is important to me. I've got everything else working quite well, its just receiving long request bodies that is still causing problems.

0 replies

espruino-discuss · 2025-02-10T19:14:48Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-20 by @allObjects

I'm not sure about the purpose of ESP8266 standalone/combined with Espruino board to be fully fledged Web (app) servers...

Yes, it is nice to have the http(s) protocol and then routing available to support modular application builds. MCs are usually used to 'just' gather some sensor's values and then transmit them in a lean way to the requester through a mediator, which complements the raw data with the shoe shine that is needed for a neat presentation. The transmitted document can be a complete html document but the 'UI-meat' comes from a third party - a UI resource server that provides the 'tons' of html(5 - html, css, js, any UI and other library/framework standard/custom code) to create a nice UI (and single page app) in the client / browser.

I'd like to keep the distributed MCs that gather the sensor data as lean as possible with the least of dependencies and changes, and then 'fully load' a classical Web (app) server with all the resources - and with CORS - it is even possible to write the data on those readily equipped servers, or flip the roles completely: deliver everything but the IoT sensor data from a regular Web (app) server and make an CORS enabled ajax call to ESP8266/Espruino to get the data.

There was a while ago this conversation about Shiny Web UI which was covering such subjects. @Snerkle, I don't have an exact understanding of all the requirements and constraints you are pursuing to satisfy, but as an MC I would be pretty overwhelmed...

In the this post I show an approach to keep the html doc sent from the IoT device as slim as possible without sacrificing the consumer UI.

In order to have it a standalone example that everyone can run just off the forum, I emulate the (temperature) sensor reading with this anonymous function in the setInterval() from lines 16 thru 23, which - when implemented - will make a slim ajax call (back) to the ESP8266/Espruino to just get the raw sensor data.

The UI library/framework I'm using in the example is dojotoolkit.org pulled in from ajax.googleapis.com. Any other smart library or framework will do too. Nice about dojo is the AMD implementation: the code in line 4 pulls only a minimum: the loader / require.js which then allows to pull into (and cache in) the browser what else is needed, as beginning with line 7.

The example the html body includes all UI layout... what makes it still a fatter 'file' then needed: putting the body content into an AMD module and load it from a regular Web (app) server - he same as one would put on the library/framework - allows even more a slimming down of what 'has to live in Espruino... which could the make an SD card obsolete... and all your related problems goners...

0 replies

gfwilliams · 2025-02-10T19:14:51Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-20 by @gfwilliams

the last binary/build that I can use is e54a6b982545da111b2f24d725fb30b0279c11da

I made some changes that should have generally increased speed/efficiency, but I tested them last night with a Pico and ESP8266 and they worked fine. And just tested them again, and they still work. Maybe it's the builds... I'll check.

It looks like the implementation is still unfinished

Any reason? Looks like I left that debug print in by accident though - it's fixed now.

Odd about the pipe ending - it's supposed to close when the close event is sent. That works when running a build on the PC, but it looks like on the Pico characters still get lost, and so the amount of data received doesn't equal content length. You could probably check that? global["\xFF"].HttpSC contains info about currently open connections - you should see cRcv in there as well as the Content-Length header.

If you run E.getErrorFlags() you might see a FIFO_FULL message - which means that the input FIFO has overrun and lost data.

can the (new) chunksize parameter be user-definable

I don't think that would help? When I looked at the data flowing through the system on my build it only seems to be pulling ~150 characters each time around the idle loop, far less than the 536 buffer size.

I'm just against adding extra complexity that realistically will never be used, especially with code size being such an issue at the moment. If you want to try stuff out, it's really not that hard to build Espruino in a VM.

req.on('end',... event

So in node.js, does res.end() close the connection immediately, or does it still wait for Content-Length bytes to be received, like is happening at the moment?

I could definitely change it to fire off an end event when content-length got reached.

Timeouts are something else that needs looking at - there's nothing implemented there at the moment at all.

0 replies

espruino-discuss · 2025-02-10T19:14:53Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-20 by Snerkle

Thanks for your interest @allObjects.
What I take from your post is that you have particular use-cases that you choose an MC for that you can make work. While I am exploring a different use-case and trying to make it work for me.

The results so far have been very fruitful and the journey has enabled me to make a number of contributions to the Espruino project in the form of bug fixes, functional improvements and some enhancements that have been readily accepted and implemented. Making Espruino better for everyone, hopefully.

So forgive me if I feel that you are suggesting I am trying to do something for which this MC isn't intended, because I disagree. Once I find its limits then I will know what use-cases I will choose this MC for, and for what use-cases I will need to choose something else.

0 replies

espruino-discuss · 2025-02-10T19:14:56Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-20 by Snerkle

Thanks @gordon, I'll try it again with the next build you post up and see if the problems are gone.

I agree with you on the buffer size, and have now observed the same thing myself.
I'll have a look at the status info and data structures you've mentioned.

I have been doing my own build's in a VM using your guide. One extra step was required from memory because recent releases of Ubuntu omit a package now for VM installs which is where the USB serial device mapping magic happens, I think the package is linux-image-extra-virtual that has to be installed or the USB serial device never shows up as /dev/tty.... So I can continue to play with buffer sizes that way if I need to.

I will look at how node behaves at res.end() and let you know.

Its a shame about the SD file writing occasionally blocking the incoming serial coms. I'd really hoped to come up with a working solution.

0 replies

gfwilliams · 2025-02-10T19:14:59Z

gfwilliams
Feb 10, 2025
Maintainer

Posted at 2016-01-20 by @gfwilliams

... just tried a build from here and that works fine too (albeit without the 'close' event).

So I'm not sure what happened for you - maybe there is a 'dead' build in there somewhere though.

I wonder what happens if you try lower baud rates now? It might be it's quite a bit more reliable.

For the VM image - please could you issue a PR for the changes in the readme file? I guess maybe a lot of people had it running previously but then never bothered to get it working with USB (they just built the image).

0 replies

espruino-discuss · 2025-02-10T19:15:01Z

espruino-discuss
Feb 10, 2025
Maintainer Author

Posted at 2016-01-20 by tve

IMHO the 'end' event would be a good addition. Right now it's somewhat painful and error prone to determine when a complete HTTP request has been received.

@allObjects: I would have to disagree with your assessment about what uC's are capable of. I totally agree that an architecture where pretty raw data from many nodes is aggregated makes a ton of sense. But while this is what I like to build myself as well, I also like to make the uC nodes that have actuators stand-alone 'cause there's nothing more frustrating than having your entire house/greenhouse/... being inoperable because the central server crashed or otherwise failed. Or not daring to upgrade the central server 'cause so much now depends on it.

What makes stand-alone http servers practical is the fact that all the UI can be shoved into browser-side JavaScript. I do that with esp-link and the constraints on the UI are pretty minimal at this point. The uC end up exchanging simple json or binary requests with a central uC and the browser alike. All that's needed is enough flash to hold the minified and compressed JS for the UI and be able to ship it to the browser. If you skip large libraries, such as jquery and bootstrap that's actually pretty easy to do.

The upshot of all this is that I'm gravitating towards uC nodes that work with a central node and that have a simple stand-alone UI.

0 replies

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Espruino JavaScript

How to go about tracking down this error? #523

{{title}}

Replies: 25 comments

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

Select a reply

Espruino JavaScript

How to go about tracking down this error? #523

espruino-discuss Feb 10, 2025 Maintainer

Replies: 25 comments

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

gfwilliams Feb 10, 2025 Maintainer

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

I'd like to propose a few things at this point:

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

espruino-discuss Feb 10, 2025 Maintainer Author

gfwilliams Feb 10, 2025 Maintainer

espruino-discuss Feb 10, 2025 Maintainer Author

espruino-discuss
Feb 10, 2025
Maintainer

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

gfwilliams
Feb 10, 2025
Maintainer

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author

espruino-discuss
Feb 10, 2025
Maintainer Author

gfwilliams
Feb 10, 2025
Maintainer

espruino-discuss
Feb 10, 2025
Maintainer Author