WebServer and Chirp Sensor (moisture, temperature, light) on ESP8266 ESP-09

[espruino-discuss]

Posted at 2018-10-29 by @allObjects

Having fiddled a bit with Espruino on ESP8266 -see sister conversation Espruino on ESP8266 ESP-09 - 1 powerful cm2 - I gave this setup a shot for to play with. Final solution will most likely more go into LoRA then Wifi connection between sensors and collector / hub station, and the data will not be pulled but pushed by the sensor node. The browser / viewer wil still do a pull, but pull will not be from the (individual) sensor(s) but from the collector / hub station. For testing things though, the intervalled pull in the browser and use of local Wifi network felt just right.

Initially I used the WebServer module... but with the setup as taken from the Espruino site - w/ inline stings and anonymous function, it stopped work after abut 8..12 pulls, and it did not reconnect. I was a bit disappointed... and lacking the energy to work myself through the WebServer module. Since I served pages before using just the http module and RYO request handler function, I did that again, as you see in the code. Additionally, I converted the http response composing to ES6 Template Strings. Then I gave the WebServer module another chance - with the inline stuff 'externalized', and now it works too... at least longer than before.

The implementation uses two (2) http requests:

1. One for the base page. It is loaded once, frames the data -in an iframe - and allows pausing and resuming the data pull.
2. One for the data pull. The data pull is just pulling a terse document into the base page as iframe.

Some trick makes the data available to the main page for further processing/rendering/etc. That I have access to the iframe document and data with .innerHTML on the node with id data, you can see in the debug window of the browser screenshot showing parts of Web page and Espruino Web IDE console. I can even JSON parse the string and get a nice object to work with.

The iframe's document (DOM) makes itself accessible to the base / main / control page by setting itself as a iframeDocument property of the top window. Initially I used a direct setting like top.iframeDocument = document; as the script in the iframe document. Now I check for the presence of the top.setIframeDocument() function / method and use that. Using a setter function provides easy hook into the event of new data for what ever (post) processing / rendering by the application. Instead of iframe, ajax / xhr call would do as well... but is way more code with no added benefits....

Using iframe to avoid ajax and make data pull as lightweight as possible was not the only reason: initially I did serve the top, framing page with `file://...``` protocol, because I was not sure if I would have enough space in the ESP-09 for all the dhtml code... but it worked itself nicely out... still about half of the variables free... ;-) ...as you can see from stats:

Connecting to wifi...
Connected to wifi.
`--- wifi.getIP(): ---`
{
  "ip": "192.168.0.119",
  "netmask": "255.255.255.0",
  "gw": "192.168.0.1",
  "mac": "18:fe:34:9c:15:3b"
 }
`--- process.env: ---`
{
  "VERSION": "2v00",
  "GIT_COMMIT": "ff35517",
  "BOARD": "ESP8266_BOARD",
  "FLASH": 0, "RAM": 81920,
  "SERIAL": "18fe349c-153b",
  "CONSOLE": "Serial1",
  "MODULES": "Flash,Storage,heatshrink,net,dgram,http,NetworkJS,Wifi,ESP8266,TelnetServer,crypto,neopixel",
  "EXPTR": 1073643636 }
`--- process.memory: ---`
{ "free": 980, "usage": 720, "total": 1700, "history": 689,
  "gc": 5, "gctime": 1.625 }

The last visual attachment shows a 404 - page not found - http error... just to verify that my 404 in the RYO WebServer does what it's supposed to to.

Here is the complete code - with static and dynamic Web resources at the end:

// E09SoilTest1And2.js
// (c)20181028 allObjects

var lg = false; // logging to console
var webServerId = "1"; // "1"=module     | "2"=RYO
                       // fails in a bit | keeps going

function log() { if (lg) console.log.apply(console,arguments); }

var wifiCreds = require("wifiCreds");
var sid = "StaEastRow3";

var wifi = require("Wifi");

function initWifi(cb) {
  log("Connecting to wifi...");
  wifi.connect(wifiCreds.ssid
      , {password:wifiCreds.pw}, function(err) {
    if (err) {
      log('error during connect:',err);
      wifi.disconnect();
    } else {
      log('Connected to wifi.');
      wifi.stopAP();
      log(wifi.getIP());
      log(process.env);
      log(process.memory());
      // wifi.save();
      setTimeout(cb,1000);
    }
  });
}

var webServer, WebServer;

function initWebServer1() {
  WebServer = require("WebServer");
  webServer = new WebServer(
    { port: 80
    , default_type:  'text/plain'
    , default_index: 'index.html'
    , memory:
      { 'info.html': { 'content': info_html, 'type': 'text/html' }
      , 'info.txt':  { 'content': 'Hello World!' }
      , 'info.njs':  { 'content': info_njs } // content from function return
      }
    });
  webServer.on('start', function (WebServer) {
    // log('WebServer listening on port ' + WebServer.port);
  });
  webServer.on('request', function (request, response, parsedUrl, WebServer) {
    // log('WebServer requested', parsedUrl);
  });
  webServer.on('error', function (error, WebServer) {
    // log('WebServer error', error);
  });
  webServer.createServer();
}


var webServer2created, webServer2listening, httpMod;

function onPageRequest(req, res) {
  var urlObj = url.parse(req.url, true), resObj;
  log(urlObj);
  switch (urlObj.pathname) {
    case "/info.html": res.writeHead(200, {'Content-Type': 'text/html'});
                       res.end(info_html); break;
    case "/info.njs" : resObj = info_njs();
                       res.writeHead(200, {'Content-Type': resObj.type });
                       res.end(resObj.content); break;
    default:           res.writeHead(404, {'Content-Type': 'text/html'});
                       res.end("");
  }
}

function initWebServer2() {
  httpMod = require("http");
  webServer2created = httpMod.createServer(onPageRequest);
  webServer2listening = webServer2created.listen(80);
}

var iidM, recM = { sid:sid, mid:0, tim:0, lite:0, temp:0, moist: 0};
var i2cM, sclM = D13, sdaM = D12; // sw I2C

function initSensorM() {
  i2cM = new I2C();
  i2cM.setup({scl:sclM, sda:sdaM});
}

function getLite(){ //takes ~3 sec. to acquire
  i2cM.writeTo(0x20,0x03);
  setTimeout(function(){
    i2cM.writeTo(0x20,0x04);
    var d = i2cM.readFrom(0x20,2);
    recM.lite = d[0]*256 + d[1];
  },3000);
}
function getTemp(){
  i2cM.writeTo(0x20,0x05);
  var d = i2cM.readFrom(0x20,2);
  recM.temp = d[0]*256 + d[1];
}
function getMoist(){
  i2cM.writeTo(0x20,0x00);
  var d = i2cM.readFrom(0x20,2);
  recM.moist = d[0]*256 + d[1];
}

function getMSensorVals() {  // done at ~:
  i2cM.writeTo(0x20,0x06);   // H+00[s] reset the sensor
  recM.mid++; recM.tim = Math.floor(getTime());
  setTimeout(getLite ,1000); // H+04
  setTimeout(getTemp ,6000); // H+06
  setTimeout(getMoist,8000); // H+08
}

function cycle() {
  getMSensorVals();
  setTimeout(function(){
      log(recM);
    },9000);
}
  
function startSensing() {
  initSensorM();
  cycle();
  iidM = setInterval(cycle,10000);
}

function onInit() {
  initWifi(function(){
    if (webServerId == "1") {
      initWebServer1();
    } else {
      initWebServer2();
    }
    startSensing();
  });
}

// static and dynamic Web resources:

var info_html = `
<html>
<!-- info.html -->
<head>
<title>Info</title>
<script>
var baseUrl = ""; // "http://192.168.0.119";
var reloadInt = 15; // reload interval in seconds
var tid, cnt = 0, paused = true;
function wait(remaining) {
  tid = null;
  if (remaining > 0) {
    document.getElementById("countDown").innerHTML
      = "" + cnt + ". Reloading in " + remaining + " seconds";
    tid = setTimeout(wait,1000,remaining - 1);
  } else {
    if (paused) {
      pause();
    } else {
      cnt++
      document.getElementById("countDown").innerHTML
        = "" + cnt + ". Loading...";
      document.getElementById("data").src = baseUrl+"/info.njs";
      wait(reloadInt);
    }
  }
}
function pause() {
  if (tid) { clearInterval(tid); tid = false; }
  paused = true;
  document.getElementById("pauseChkBox").checked = true;
  document.getElementById("countDown").innerHTML = "Paused";
}
function resume() {
  paused = false;
  document.getElementById("pauseChkBox").checked = false;
  wait(0);
}
function setIframeDocument(iframeDocument) {
  top.iframeDocument = iframeDocument;
  // ...and some other code using it
}
</script>
</head>
<body onload="wait(0);">
<h3>Info</h3>
<div><p>
  <input id="pauseChkBox" type="checkbox"
    onclick="if (this.checked) { pause(); } else { resume(); }" 
    > Pause
</p></div>
<div><p><span id="countDown"></span><p></div>
<div><iframe id="data" style="width:100%;"></iframe></div>
</body>
</html>
`;

function info_njs() {
  return {'type': 'text/html', 'content': `
<html><body><div id="info">${JSON.stringify(recM)}</div>
<script>if (top.setIframeDocument) top.setIframeDocument(document);
</script></body></html>
` };
}

setTimeout(onInit,1000); // to be removed for save()

---

Attachments:

• ESP09WS1.png
  
• E09EChirp.jpg
  
• Nice404PageNotFound.png

[espruino-discuss]

Posted at 2018-10-29 by @allObjects

...still dying after a while with:

 ets Jan  8 2013,rst cause:1, boot mode:(3,6)
load 0x40100000, len 2408, room 16
tail 8
chksum 0xe5
load 0x3ffe8000, len 776, room 0
tail 8
chksum 0x84
load 0x3ffe8310, len 632, room 0
tail 8
chksum 0xd8
csum 0xd8
2nd boot version : 1.6
  SPI Speed      : 40MHz
  SPI Mode       : QIO
  SPI Flash Size & Map: 8Mbit(512KB+512KB)
jump to run user1 @ 1000
�ã�ì�gäÛs�ò'|ì�lÇxsd�#���<�sÛ�d�'à�'ãd`��ãrÛl�dþLoading 1219 bytes from flash...
>

It's a restart, but have to figure out why this Espruino on a ESP8266 happends. But before that though, I will run same app in the configuration of an Espruino with an ESP8266.

---

Posted at 2018-10-29 by Wilberforce

Have a look at your free vars and also free heap. I would say you are running out of memory. The esp8266 is quite limited in ram.

---

Posted at 2018-10-29 by @allObjects

@wilberforce, did so after start: process.memory(): { "free": 980, "usage": 720, "total": 1700, "history": 689, "gc": 5, "gctime": 1.625 }... but I will have to do while going on, because that's what I suspect that along the processing of requests it could leak. On the other hand, when run first, only WebServer got stuck, the reading of the sensor in intervals kept going...

---

Posted at 2018-10-30 by @MaBecker

Please check your ESP8266 state in function cycle() to check if you are running out of heap space

require('ESP8266').getState();
={
  sdkVersion: "2.2.1(6ab97e9)",
  cpuFrequency: 160, freeHeap: 10624, maxCon: 10,
  flashMap: "512KB:256/256",
  flashKB: 512,
  flashChip: "0xe0 0x4016"
 }

---

Posted at 2018-10-30 by @MaBecker

freeHeap is the one you should watch

---

Posted at 2018-10-30 by @allObjects

Free heap started out at abut 9600... went down to 7200, then backup and circling around 8800, and then, on the 50th - server side - cycle and on the 33rd - client/browser side cycle (pull data) - it crashed... with heap still 7720... and after crashing - without touching - it keeps crashing.... and crashing... and crashing,... about every minute once... with same message.

{
  "sdkVersion": "2.2.1(6ab97e9)",
  "cpuFrequency": 160, "freeHeap": 7720, "maxCon": 10,
  "flashMap": "1MB:512/512",
  "flashKB": 1024,
  "flashChip": "0xef 0x4014"
 }
{
  "sid": "StaWestRow3",
  "mid": 49, "tim": 1540900784, "lite": 0, "temp": 0, "moist": 0 }
>
 ets Jan  8 2013,rst cause:1, boot mode:(3,7)
load 0x40100000, len 2408, room 16
tail 8
chksum 0xe5
load 0x3ffe8000, len 776, room 0
tail 8
chksum 0x84
load 0x3ffe8310, len 632, room 0
tail 8
chksum 0xd8
csum 0xd8
2nd boot version : 1.6
  SPI Speed      : 40MHz
  SPI Mode       : QIO
  SPI Flash Size & Map: 8Mbit(512KB+512KB)
jump to run user1 @ 1000
�ã�ä��oì�r�ûg|ì�$�prl�cÛ��|�s��d�gàÃgãd`��ã;�d�d�þLoading 1219 bytes from flash...
> 

The code as below can be run in any environment with isEspruino = true / false; set accordingly for Espruino-Wifi (or Espruino with a serially connected ESP8266 for Wifi0) and for ESP8266 (Espruino on ESP8266). The sensor (isChirp) is set to false and thus is not talked to... only the counting of cycles happens and all Espruino/ESP8266/server-side and data-pulling/browser/client-side mechanics. Browser url: <ipAddress>/info.html, and setting on page: Pause unchecked. Eventhough IP address is not available (anymore) when asked for after connect w/ hostname set, a bit later it can be asked for in the console with wifi.getIP().

The code I'm running now is this (with the anomaly that after setting the hostname, I get weird result w/ wifi.getIP(...): ip: 0.0.0.0 - and mentioned ips specific to my wireless LAN):

// SoilWifiPullTest.js
// (c)20181029 allObjects
// for ip, turn log on and check.
// http://192.168.0.119/info.html EonESP
// http://192.168.0.121/info.html EWifi

var lg = true;
var sid = "StaWestRow3";
var isChirp = false;
var isEspruino = false;
var hostname = "westrow3";
var webServerId = "2"; // "1"=module |　"2"=RYO

function log() { if (lg) console.log.apply(console,arguments); }

var wifiCreds = require("wifiCreds");

var wifi = require("Wifi"); // EWifi, EonE

function initWifi(cb) {

      wifi.setHostname(hostname,function(err){
        if (err) {
          log(`error on setHostname: ${err}`);
        } else {
          log(`Hostname ${hostname} set.`);

  log(`Connecting ${hostname} to wifi...`);
  wifi.connect(wifiCreds.ssid
      , {password:wifiCreds.pw}, function(err) {
    if (err) {
      log(`error on connecting ${hostname}: ${err}`);
      wifi.disconnect();
    } else {
      log(`Connected ${hostname} to wifi.`);
      wifi.getIP(function(err,ip){
        if (err) { // ...ever in ESP8266?
          log(`error on getIP for ${hostname}: ${err}`);
          wifi.disconnect();
        } else {
          if (ip.ip == "0.0.0.0") {
            log("Espruino on ESP8266");
            wifi.stopAP();
            ip = wifi.getIP();
          }
          log(ip);
          log(process.env);
          log(process.memory());
          cb();
        }
      });
    }
  });

        }
      });

}


var webServer, WebServer;

function initWebServer1() {
  WebServer = require("WebServer");
  webServer = new WebServer(
    { port: 80
    , default_type:  'text/plain'
    , default_index: 'index.html'
    , memory:
      { 'info.html': { 'content': info_html, 'type': 'text/html' }
      , 'info.txt':  { 'cowestrow3ntent': 'Hello World!' }
      , 'info.njs':  { 'content': info_njs } // content from function return
      }
    });
  webServer.on('start', function (WebServer) {
    // log('WebServer listening on port ' + WebServer.port);
  });
  webServer.on('request', function (request, response, parsedUrl, WebServer) {
    // log('WebServer requested', parsedUrl);
  });
  webServer.on('error', function (error, WebServer) {
    // log('WebServer error', error);
  });
  webServer.createServer();
}


var webServer2created, webServer2listening, httpMod;

function onPageRequest(req, res) {
  var urlObj = url.parse(req.url, true), resObj;
  log(urlObj);
  switch (urlObj.pathname) {
    case "/info.html": res.writeHead(200, {'Content-Type': 'text/html'});
                       res.end(info_html); break;
    case "/info.njs" : resObj = info_njs();
                       res.writeHead(200, {'Content-Type': resObj.type });
                       res.end(resObj.content); break;
    default:           res.writeHead(404, {'Content-Type': 'text/html'});
                       res.end("");
  }
}

function initWebServer2() {
  httpMod = require("http");
  webServer2created = httpMod.createServer(onPageRequest);
  webServer2listening = webServer2created.listen(80);
}

var iidM, recM = { sid:sid, mid:0, tim:0, lite:0, temp:0, moist: 0};
var i2cM, sclM, sdaM;
if (isEspruino) {
  i2cM = I2C2; sclM = B10; sdaM = B3; // hw I2C EwE
} else {
  i2cM = null; sclM = D13; sdaM = D12; // sw I2C EonE
}

function initSensorM() {
  i2cM = i2cM || new I2C();
  i2cM.setup({scl:sclM, sda:sdaM});
}

function getLite(){ // takes ...3[s] to acquire
  i2cM.writeTo(0x20,0x03);
  setTimeout(function(){
    i2cM.writeTo(0x20,0x04);
    var d = i2cM.readFrom(0x20,2);
    recM.lite = d[0]*256 + d[1];
  },3000);
}
function getTemp(){
  i2cM.writeTo(0x20,0x05);
  var d = i2cM.readFrom(0x20,2);
  recM.temp = d[0]*256 + d[1];
}
function getMoist(){
  i2cM.writeTo(0x20,0x00);
  var d = i2cM.readFrom(0x20,2);
  recM.moist = d[0]*256 + d[1];
}

function getMSensorVals() {  // done at ~:
  recM.mid++; recM.tim = Math.floor(getTime());
  if (isChirp) {
    i2cM.writeTo(0x20,0x06);   // H+00[s] reset the sensor
    setTimeout(getLite ,1000); // H+01..04 (timed read 3[s] after cmd)
    setTimeout(getTemp ,6000); // H+06..
    setTimeout(getMoist,8000); // H+08..
  }                            // H+09      log will be issued
}

function cycle() {
  log(require('ESP8266').getState());
  getMSensorVals();
  setTimeout(function(){ log(recM); },9000);
}

function startSensing() {
  if (isChirp) initSensorM();
  cycle();
  iidM = setInterval(cycle,10000);
}

function onInit() {
  initWifi(function(){
    if (webServerId == "1") {
      initWebServer1();
    } else {
      initWebServer2();
    }
    startSensing();
  });
}

// static and dynamic Web resources:

var info_html = `
<html>
<!-- info.html -->
<head>
<title>Info</title>
<script>
var baseUrl = "";
var reloadInt = 15; // reload interval in seconds
var tid, datTim = "...", cnt = 0, paused = true;
function wait(remaining) {
  tid = null;
  if (remaining > 0) {
    document.getElementById("countDown").innerHTML
      = ""+cnt+". "+datTim+" : Reloading in "+remaining+" seconds.";
    tid = setTimeout(wait,1000,remaining - 1);
  } else {
    if (paused) {
      pause();
    } else {
      cnt++
      document.getElementById("countDown").innerHTML
        = ""+cnt+". Loading...";
      document.getElementById("data").src = baseUrl+"/info.njs";
      wait(reloadInt);
    }
  }
}
function pause() {
  if (tid) { clearInterval(tid); tid = false; }
  paused = true;
  document.getElementById("pauseChkBox").checked = true;
  document.getElementById("countDown").innerHTML = "Paused";
}
function resume() {
  paused = false;
  document.getElementById("pauseChkBox").checked = false;
  wait(0);
}
function setIframeDocument(iframeDocument) {
  top.iframeDocument = iframeDocument;
  var info = iframeDocument.getElementById("info").innerHTML;
  var recM = JSON.parse(info);
  datTim = new Date(recM.tim * 1000).toISOString()
  datTim = datTim.substr(0,19).replace("T"," ");
  // ...and some other code using it
}
</script>
</head>
<body onload="wait(0);">
<h3>Info</h3>
<div><p>
  <input id="pauseChkBox" type="checkbox"
    onclick="if (this.checked) { pause(); } else { resume(); }" 
    > Pause
</p></div>
<div><p><span id="countDown"></span><p></div>
<div><iframe id="data" style="width:100%;"></iframe></div>
</body>
</html>
`;

function info_njs() {
  return {'type': 'text/html', 'content': `
<html><body><div id="info">${JSON.stringify(recM)}</div>
<script>if (top.setIframeDocument) top.setIframeDocument(document);
</script></body></html>
` };
}

setTimeout(onInit,1000);

---

Posted at 2018-10-30 by @MaBecker

Hmm, let me try this with a debug enabled build.

---

Posted at 2018-11-05 by maman

Hello @allObjects,

What is espruino version do you use ?
Previously, I used espruino_2v00.17 on ESP-07. It was periodically crashed.
Changed to espruino_1v99, using the same script code, it is no problem.

---

Posted at 2018-11-05 by @allObjects

@maman, thank you for this information. I will give that a try. Yes, I used 2v00, revision not known at this point, but it is ESP-09 - zero nine... Even though I think it should not really matter, because it is all the same EX chip, there could could be some difference. I have also the ESP-07S -zero - seven - S..., which I will try now for sure w/ Espruino on it. I use it though for now only as plain ESP8266 over serial / AT protocol.