ESP32-S2 keep reboot while flashing firmware from HTTP GET from github.

[VaAndCob]

Board

ESP32-S2 Lolin Mini V1.00

Device Description

ESP32-S2 Lolin Mini V1.0.0

Hardware Configuration

No connection , just bare ESP32-S2 Lolin Mini board

Version

latest master (checkout manually)

IDE Name

ARDUINO IDE 2.2.1

Operating System

macOS Montery 12.7.1 , Windows 10

Flash frequency

80 Mhz

PSRAM enabled

yes

Upload speed

921600

Description

If testes OTA firmware update from HTTP. it works very well on ESP32-S2 Lolin Mini,

BUT..... when I copy this code and combine with another Arduino sketch ( to add HTTP OTA firmware update capability)
the update progress runs until 22-25% approximately and then reboots itself without any "EXCEPTION, BACKTRACE, or error message printed on serial."
Can someone please guide me what is the cause?

• the same hardware,
• the same sketch,
• only add this OTA sketch to another working sketch.

I checked the stack and heap. still available during flashing. so I don't think it's running out of memory issue.

Sketch

const char* ssid = "WIFI_2.4G";  
const char* password = "passwords"; // your network password

// S3 Bucket Config
const String firmwareURL = "https://myrepo.github.io/firmware/FW1_2_1.bin"; // Host => bucket-name.s3.region.amazonaws.com
#include <WiFi.h>
#include <HTTPClient.h>
#include <Update.h>
//add comma to number and change to string
String formatNumberWithCommas(uint32_t number) {
  String formatted;
  int count = 0;
  do {
    if (count > 0 && count % 3 == 0) {
      formatted = "," + formatted;
    }
    formatted = String(number % 10) + formatted;
    number /= 10;
    count++;
  } while (number > 0);
  return formatted;
}

void performOnlineUpdate() { //download file from github and update
    String freeheap = formatNumberWithCommas(ESP.getFreeHeap());
  String freestack = formatNumberWithCommas(uxTaskGetStackHighWaterMark(NULL));
  Serial.println("Free Mem  : (Heap/Stack)");
  int last_progress = 0;
  HTTPClient client;
  client.begin(firmwareURL);

  int httpCode = client.GET();//request file

  if (httpCode == HTTP_CODE_OK) {//200
    uint32_t fileSize = client.getSize();
    Serial.printf("Firmware Size: %d bytes\n", fileSize);

    if (Update.begin(fileSize)) {//start updating

      uint8_t buff[1024] = {0};    // create buffer for read

      WiFiClient *stream = client.getStreamPtr();    // get tcp stream
    
      Update.onProgress([&](size_t progress, size_t total) {//get update progress
        int curr_progress = round(progress / (total /100));
        if (curr_progress != last_progress) {//if progress not change then skip update progress bar
          String progressString = String(curr_progress);
          last_progress = curr_progress;
          Serial.println(progressString+" %  ");
          String heap = formatNumberWithCommas(ESP.getFreeHeap())+"/"+freeheap;
          String stack = formatNumberWithCommas(uxTaskGetStackHighWaterMark(NULL))+"/"+freestack;
          Serial.println(heap+" - "+stack);
        }
      });


      while (client.connected() && (fileSize > 0)) {
        size_t size = stream->available();// get available data size
        if (size) {// read up to 1024 byte
         int c = stream->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size));
         Update.write(buff, c);
         if (fileSize > 0) fileSize -= c;//calculate remaining byte
        }//if size
        delay(1);
      }
      if (fileSize == 0) {
        Update.end(true);
        Serial.println(F("Update Successful... Restart"));
        delay(3000);
        ESP.restart();
      }
    } else {//update.begin
      Serial.println("⚠️ Error: Not enough space!");
    }
   
  } else {//error not 200
    Serial.printf("HTTP request failed. HTTP code: %d\n", httpCode);

  }
    client.end();
  
} 
//------------------

void setup() {

  Serial.begin(115200);

   while (!Serial) {}
  pinMode(15,OUTPUT);
  // Connect to Wi-Fi
  WiFi.disconnect();
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(1000);
    Serial.println("Connecting to WiFi...");
  }
  Serial.println("Connected to WiFi");
  WiFi.setTxPower(WIFI_POWER_2dBm); //set wifi tx power to default
  delay(5000);

  performOnlineUpdate();
}

void loop() {
yield();  // Your main code here
}

Debug Message

[ 82725][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Content-Type: application/octet-stream'
[ 82734][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'permissions-policy: interest-cohort=()'
[ 82743][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Last-Modified: Mon, 27 Nov 2023 05:40:34 GMT'
[ 82753][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Access-Control-Allow-Origin: *'
[ 82760][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Strict-Transport-Security: max-age=31556952'
[ 82769][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'ETag: "65642bd2-12ab10"'

[ 82776][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'expires: Wed, 29 Nov 2023 13:10:24 GMT'
[ 82785][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Cache-Control: max-age=600'
[ 82794][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'x-proxy-cache: MISS'
[ 82801][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'X-GitHub-Request-Id: F976:1BBD92:10AB9A:12D136:656735E7'

[ 82811][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Accept-Ranges: bytes'
[ 82817][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Date: Wed, 29 Nov 2023 13:19:10 GMT'
[ 82825][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Via: 1.1 varnish'
[ 82832][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Age: 0'
[ 82839][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'X-Served-By: cache-bkk2310023-BKK'
[ 82847][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'X-Cache: HIT'
[ 82855][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'X-Cache-Hits: 0'
[ 82862][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'X-Timer: S1701263950.867365,VS0,VE327'
[ 82870][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'Vary: Accept-Encoding'
[ 82877][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: 'X-Fastly-Request-ID: 3d930610680e0bf75f0b8ec0726ef99089fd2f31'
[ 82887][V][HTTPClient.cpp:1264] handleHeaderResponse(): RX: ''
[ 82893][D][HTTPClient.cpp:1321] handleHeaderResponse(): code: 200
[ 82899][D][HTTPClient.cpp:1324] handleHeaderResponse(): size: 1223440
[ 82904][D][HTTPClient.cpp:642] sendRequest(): sendRequest code=200

Firmware Size: 1223440 bytes
[ 82910][D][Updater.cpp:133] begin(): OTA Partition: app1
     free stack      free heap      (during OTA / before http GET)
1 %  32,744/80,896 - 11,876/13,740
2 %  32,884/80,896 - 11,876/13,740
3 %  32,712/80,896 - 11,876/13,740
4 %  32,468/80,896 - 11,876/13,740
5 %  32,676/80,896 - 11,876/13,740
6 %  32,432/80,896 - 11,876/13,740
7 %  32,884/80,896 - 11,876/13,740
8 %  32,712/80,896 - 11,876/13,740
9 %  32,712/80,896 - 11,876/13,740
10 %  32,716/80,896 - 11,876/13,740
11 %  32,680/80,896 - 11,876/13,740
12 %  32,420/80,896 - 11,876/13,740
13 %  32,628/80,896 - 11,876/13,740
14 %  32,884/80,896 - 11,876/13,740
15 %  32,924/80,896 - 11,876/13,740
16 %  32,464/80,896 - 11,876/13,740
17 %  32,884/80,896 - 11,876/13,740
18 %  32,880/80,896 - 11,876/13,740
19 %  32,712/80,896 - 11,876/13,740
20 %  32,504/80,896 - 11,876/13,740
21 %  32,712/80,896 - 11,876/13,740

and reboot at 21%

Other Steps to Reproduce

No response

I have checked existing issues, online documentation and the Troubleshooting Guide

• I confirm I have checked existing issues, online documentation and Troubleshooting guide.

[lbernstone]

Does it work if you take out all the calls to formatNumber....? Passing around strings like that is a very bad idea and leads to memory leaks.

[VaAndCob]

Does it work if you take out all the calls to formatNumber....? Passing around strings like that is a very bad idea and leads to memory leaks.

It didn't work, so formatNumber function is for debugging purposes only, I want to know if it's memory related, usually, I don't put that function and no memory checks during OTA.

[Jason2866]

We saw sometimes failing in OTA updates for unicore MCUs too. We solved by adding a delay(1) in line 396 in Updater.cpp

        _bufferLen += toRead;
        if((_bufferLen == remaining() || _bufferLen == SPI_FLASH_SEC_SIZE) && !_writeBuffer())
            return written;
        written += toRead;

        delay(1);  // Fix unicore WDT
    }
    return written;
}

and in esp.cpp line 252-254

        md5.add(buf.get(), readBytes);
        lengthLeft -= readBytes;
        offset += readBytes;

	#if CONFIG_FREERTOS_UNICORE
        delay(1);  // Fix solo WDT
        #endif
    }
    md5.calculate();
    result = md5.toString();
    return result;
}

[VaAndCob]

We saw sometimes failing in OTA updates for unicore MCUs too. We solved by adding a delay(1) in line 396 in Updater.cpp

        _bufferLen += toRead;
        if((_bufferLen == remaining() || _bufferLen == SPI_FLASH_SEC_SIZE) && !_writeBuffer())
            return written;
        written += toRead;

        delay(1);  // Fix unicore WDT
    }
    return written;
}

and in esp.cpp line 252-254

        md5.add(buf.get(), readBytes);
        lengthLeft -= readBytes;
        offset += readBytes;

	#if CONFIG_FREERTOS_UNICORE
        delay(1);  // Fix solo WDT
        #endif
    }
    md5.calculate();
    result = md5.toString();
    return result;
}

I edited both files, unfortunately. not help. thank u anyway.

I just wonder if there is anyone who experiences firmware OTA updates and reboots during flashing... especially in large firmware size
. I'm sure my partition is minimal SPIFF 1.9 MB App 195 kb SPIFF. So large enough for a firmware file size of 1.2 MB.

[lbernstone]

Is the location consistently at 256k transferred?

[VaAndCob]

Is the location consistently at 256k transferred?

Excuse me. what do u mean? Do u mean esp32-s2 crash and reboot at 256k transferred? let me check.

22 %
270336 bytes
Crashed at 270336 bytes written. seem close to 256k bytes transferred.....

[VaAndCob]

I had tested for several times, exactly crashed at
21 %
258048 bytes
262144 bytes
266240 bytes
22 %
270336 bytes -> crashed -> reboot

[lbernstone]

No, if it was actually at 256k I would be thinking some buffer might be exhausted, but 270k is nothing special.
It is completely crashing? You get the cold boot sequence?

[VaAndCob]

No, if it was actually at 256k I would be thinking some buffer might be exhausted, but 270k is nothing special. It is completely crashing? You get the cold boot sequence?

yes, it boots normally with the current firmware without any problems. there's nothing changed.

I wonder. why the sketch above works well and completes 100% OTA flashing without any problems, but when I combine this OTA sketch with another main sketch (to add the OTA feature). Crashing during writing flash occurred

[Jason2866]

One thing, how does it come that an http connection is possible to github?
Afaik github only accepts https. Are you sure it is transfered what you think it should?

[VaAndCob]

One thing, how does it come that an http connection is possible to github?
Afaik github only accepts https. Are you sure it is transfered what you think it should?

Yeah it worked perfectly. You can try sketch above with github file but correct url

[lbernstone]

Shut down any other tasks you have going on during update, particularly anything using SPI or flash. You are hoping to reboot at the end of the update, so you should be preparing for that anyhow.

[VaAndCob]

Shut down any other tasks you have going on during update, particularly anything using SPI or flash. You are hoping to reboot at the end of the update, so you should be preparing for that anyhow.

I stopped all tasks and loops by putting the "updating flag" , only the OTA task but gave the same result.

Not sure if it's about Async Web Server?
in the full sketch, esp32 runs an async web server. and can update "local" and "online" as the sketch below.

• Local update - browse the firmware file that is downloaded to a device and update. -> work without problem.
• Online Update - direct download the firmware file from Github and update. -> Stuck at 270366 bytes flash.

`

//handle online firmware updating from github
server.on("/onlineupdate", HTTP_GET, [](AsyncWebServerRequest *request){

    if (update_available) {
      request->send_P(200, "text/plain", "🔄 Updating... please wait");
      performOnlineUpdate();//perform updaet if new firmware avaialble 
    } else {
      Serial.println(F("-> Firmware is up to date"));
      request->send_P(200, "text/plain", "✅ Firmware is up to date");  
    }//update avaialble
  });

 //handle local file firmware updating
  server.on("/localupdate", HTTP_POST, [](AsyncWebServerRequest *request){
    //request->send(200, "text/plain", (Update.hasError()) ? "FAIL" : "OK");
    if (Update.hasError()) { //if update error occured
      request->send_P(200, "text/plain", "⚠️ Error: bad firmware file!");
      Serial.println("⚠️ Error: bad firmware file!");
    } else { //if no error
      request->send_P(200, "text/plain", "✅ Update Successful... Restart");
     esp_ota_mark_app_valid_cancel_rollback(); // Prevent from booting the old firmware
     delay(3000);
     ESP.restart(); // Successful upload, restart ESP
    }
  }, [](AsyncWebServerRequest *request, String filename, size_t index, uint8_t *data, size_t len, bool final){
     if (!index) {
       Serial.printf("Firmware: %s\n", filename.c_str());
       Serial.println(F("🔄 Updating... please wait"));
       if (!Update.begin(UPDATE_SIZE_UNKNOWN)) {
        Update.printError(Serial);
       }
    }
    if (Update.write(data, len) != len) { //perform update firmware
      Update.printError(Serial);
    }
    if (final) {
      if (Update.end(true)) { // true to set the size to the current progress
        Serial.println(F("Update Successful... Restart"));
      } else {
        Update.printError(Serial);
      }
   }
  });`

[VaAndCob]

report the result. After trying remove sketch parts one by one. I found that.
server.on event handler is the problem.
firmware OTA update must not be inside server.on event handler