From 3849a253f23b6f86c6f79be3f7be303471901876 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Mon, 3 Feb 2025 12:57:34 +0000 Subject: [PATCH 1/7] v2.10 - restructure serial logging --- Firmware/OpenLog_Artemis/OpenLog_Artemis.ino | 109 +++++++++++-------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/Firmware/OpenLog_Artemis/OpenLog_Artemis.ino b/Firmware/OpenLog_Artemis/OpenLog_Artemis.ino index a5ba62b..5a5a034 100644 --- a/Firmware/OpenLog_Artemis/OpenLog_Artemis.ino +++ b/Firmware/OpenLog_Artemis/OpenLog_Artemis.ino @@ -163,10 +163,13 @@ v2.9 Adds support for TMP102 low(er) cost temperature sensor + + v2.10 + Restructure the serial logging code in loop() */ const int FIRMWARE_VERSION_MAJOR = 2; -const int FIRMWARE_VERSION_MINOR = 9; +const int FIRMWARE_VERSION_MINOR = 10; //Define the OLA board identifier: // This is an int which is unique to this variant of the OLA and which allows us @@ -176,7 +179,7 @@ const int FIRMWARE_VERSION_MINOR = 9; // the variant * 0x100 (OLA = 1; GNSS_LOGGER = 2; GEOPHONE_LOGGER = 3) // the major firmware version * 0x10 // the minor firmware version -#define OLA_IDENTIFIER 0x128 // Stored as 296 decimal in OLA_settings.txt +#define OLA_IDENTIFIER 0x12A // Stored as 298 decimal in OLA_settings.txt //#define noPowerLossProtection // Uncomment this line to disable the sleep-on-power-loss functionality @@ -291,7 +294,6 @@ Apollo3RTC myRTC; //Create instance of RTC class uint64_t lastSeriaLogSyncTime = 0; uint64_t lastAwakeTimeMillis; const int MAX_IDLE_TIME_MSEC = 500; -bool newSerialData = false; char incomingBuffer[256 * 2]; //This size of this buffer is sensitive. Do not change without analysis using OpenLog_Serial. int incomingBufferSpot = 0; int charsReceived = 0; //Used for verifying/debugging serial reception @@ -611,39 +613,19 @@ void loop() { if (settings.logSerial == true && online.serialLogging == true && settings.useTxRxPinsForTerminal == false) { - size_t timestampCharsLeftToWrite = strlen(serialTimestamp); - //SerialPrintf2("timestampCharsLeftToWrite is %d\r\n", timestampCharsLeftToWrite); - //SerialFlush(); + // v2.10 - this code has been restructured.... - if (Serial1.available() || (timestampCharsLeftToWrite > 0)) + // The writing of the timestamp has the highest priority + if (strlen(serialTimestamp) > 0) { - while (Serial1.available() || (timestampCharsLeftToWrite > 0)) + while (strlen(serialTimestamp) > 0) // Copy all timestamp chars into incomingBuffer { - if (timestampCharsLeftToWrite > 0) // Based on code written by @DennisMelamed in PR #70 - { - incomingBuffer[incomingBufferSpot++] = serialTimestamp[0]; // Add a timestamp character to incomingBuffer + incomingBuffer[incomingBufferSpot++] = serialTimestamp[0]; // Add a timestamp character to incomingBuffer - for (size_t i = 0; i < timestampCharsLeftToWrite; i++) - { - serialTimestamp[i] = serialTimestamp[i+1]; // Shuffle the remaining chars along by one, including the NULL terminator - } - - timestampCharsLeftToWrite -= 1; // Now decrement timestampCharsLeftToWrite - } - else + size_t timestampCharsLeftToWrite = strlen(serialTimestamp); + for (size_t i = 0; i < timestampCharsLeftToWrite; i++) { - incomingBuffer[incomingBufferSpot++] = Serial1.read(); - charsReceived++; - - //Get the RTC timestamp if we just received the timestamp token - if (settings.timestampSerial && (incomingBuffer[incomingBufferSpot-1] == settings.timeStampToken)) - { - getTimeString(&serialTimestamp[2]); - serialTimestamp[0] = 0x0A; // Add Line Feed at the start of the timestamp - serialTimestamp[1] = '^'; // Add an up-arrow to indicate the timestamp relates to the preceeding data - serialTimestamp[strlen(serialTimestamp) - 1] = 0x0A; // Change the final comma of the timestamp to a Line Feed - timestampCharsLeftToWrite = strlen(serialTimestamp); // Update timestampCharsLeftToWrite now, so timestamp is printed immediately (#192) - } + serialTimestamp[i] = serialTimestamp[i+1]; // Shuffle the remaining chars along by one, including the NULL terminator } if (incomingBufferSpot == sizeof(incomingBuffer)) @@ -652,35 +634,68 @@ void loop() { serialDataFile.write(incomingBuffer, sizeof(incomingBuffer)); //Record the buffer to the card digitalWrite(PIN_STAT_LED, LOW); incomingBufferSpot = 0; + + //If we are sleeping between readings then we cannot rely on millis() as it is powered down + //Use RTC instead + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + + checkBattery(); } - checkBattery(); } - - //If we are sleeping between readings then we cannot rely on millis() as it is powered down - //Use RTC instead - lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now - newSerialData = true; } - else if (newSerialData == true) + + // Now check for incoming serial data. Process bytes until a timestamp token is detected + if (Serial1.available()) { - if ((rtcMillis() - lastSeriaLogSyncTime) > MAX_IDLE_TIME_MSEC) //If we haven't received any characters recently then sync log file + while ((Serial1.available()) && (strlen(serialTimestamp) == 0)) { - if (incomingBufferSpot > 0) + incomingBuffer[incomingBufferSpot++] = Serial1.read(); + charsReceived++; + + //Get the RTC timestamp if we just received the timestamp token + if (settings.timestampSerial && (incomingBuffer[incomingBufferSpot-1] == settings.timeStampToken)) + { + getTimeString(&serialTimestamp[2]); + serialTimestamp[0] = 0x0A; // Add Line Feed at the start of the timestamp + serialTimestamp[1] = '^'; // Add an up-arrow to indicate the timestamp relates to the preceeding data + serialTimestamp[strlen(serialTimestamp) - 1] = 0x0A; // Change the final comma of the timestamp to a Line Feed + } + + if (incomingBufferSpot == sizeof(incomingBuffer)) { - //Write the remainder of the buffer digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording - serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card - serialDataFile.sync(); - if (settings.frequentFileAccessTimestamps == true) - updateDataFileAccess(&serialDataFile); // Update the file access time & date + serialDataFile.write(incomingBuffer, sizeof(incomingBuffer)); //Record the buffer to the card digitalWrite(PIN_STAT_LED, LOW); - incomingBufferSpot = 0; + + //If we are sleeping between readings then we cannot rely on millis() as it is powered down + //Use RTC instead + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + + checkBattery(); } + } + } + + // Periodically sync data to SD + if ((rtcMillis() - lastSeriaLogSyncTime) > MAX_IDLE_TIME_MSEC) //If we haven't logged any characters recently then sync log file + { + if (incomingBufferSpot > 0) + { + //Write the remainder of the buffer + digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording + serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card + serialDataFile.sync(); + if (settings.frequentFileAccessTimestamps == true) + updateDataFileAccess(&serialDataFile); // Update the file access time & date + digitalWrite(PIN_STAT_LED, LOW); + + incomingBufferSpot = 0; - newSerialData = false; lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now printDebug("Total chars received: " + (String)charsReceived + "\r\n"); + + checkBattery(); } } } From a93b2d61d999dfd79069a5c9125fd5b2396e426a Mon Sep 17 00:00:00 2001 From: PaulZC Date: Mon, 3 Feb 2025 13:04:04 +0000 Subject: [PATCH 2/7] actions/upload-artifact@v4 --- .github/workflows/build-for-release.yml | 2 +- .github/workflows/non-release-build.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-for-release.yml b/.github/workflows/build-for-release.yml index a14663f..a9f438e 100644 --- a/.github/workflows/build-for-release.yml +++ b/.github/workflows/build-for-release.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@master + uses: actions/checkout@main - name: Extract branch name run: echo "BRANCH=${{github.ref_name}}" >> $GITHUB_ENV diff --git a/.github/workflows/non-release-build.yml b/.github/workflows/non-release-build.yml index 01616cf..2902797 100644 --- a/.github/workflows/non-release-build.yml +++ b/.github/workflows/non-release-build.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@master + uses: actions/checkout@main - name: Get firmware version 1 run: | @@ -108,7 +108,7 @@ jobs: echo "targetBinary=./Firmware/${{ env.FILENAME_PREFIX }}/build/SparkFun.apollo3.sfe_artemis_atp/${{ env.FILENAME_PREFIX }}${{ env.firmwareMajorMinor }}.bin" >> "$GITHUB_ENV" - name: Upload binary to action - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.FILENAME_PREFIX }}${{ env.firmwareMajorMinor }}.bin path: ${{ env.targetBinary }} From 2da1dc8a71cdb48fd814c933d1461beb6868dae0 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 6 Feb 2025 13:12:50 +0000 Subject: [PATCH 3/7] Where possible, write residual serial data to file before closing --- Firmware/OpenLog_Artemis/OpenLog_Artemis.ino | 13 +++++++ Firmware/OpenLog_Artemis/lowerPower.ino | 36 +++++++++++++++++++ Firmware/OpenLog_Artemis/menuMain.ino | 24 +++++++++++++ .../OpenLog_Artemis/menuSerialLogging.ino | 12 +++++++ 4 files changed, 85 insertions(+) diff --git a/Firmware/OpenLog_Artemis/OpenLog_Artemis.ino b/Firmware/OpenLog_Artemis/OpenLog_Artemis.ino index 5a5a034..49a5bec 100644 --- a/Firmware/OpenLog_Artemis/OpenLog_Artemis.ino +++ b/Firmware/OpenLog_Artemis/OpenLog_Artemis.ino @@ -166,6 +166,7 @@ v2.10 Restructure the serial logging code in loop() + Where possible, write residual serial data to file before closing */ const int FIRMWARE_VERSION_MAJOR = 2; @@ -812,6 +813,18 @@ void loop() { } if (online.serialLogging == true) { + if (incomingBufferSpot > 0) + { + //Write the remainder of the buffer + digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording + serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card + digitalWrite(PIN_STAT_LED, LOW); + + incomingBufferSpot = 0; + + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + } + serialDataFile.sync(); updateDataFileAccess(&serialDataFile); // Update the file access time & date serialDataFile.close(); diff --git a/Firmware/OpenLog_Artemis/lowerPower.ino b/Firmware/OpenLog_Artemis/lowerPower.ino index f582109..7bef8dd 100644 --- a/Firmware/OpenLog_Artemis/lowerPower.ino +++ b/Firmware/OpenLog_Artemis/lowerPower.ino @@ -183,6 +183,18 @@ void resetArtemis(void) } if (online.serialLogging == true) { + if (incomingBufferSpot > 0) + { + //Write the remainder of the buffer + digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording + serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card + digitalWrite(PIN_STAT_LED, LOW); + + incomingBufferSpot = 0; + + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + } + serialDataFile.sync(); updateDataFileAccess(&serialDataFile); // Update the file access time & date serialDataFile.close(); @@ -281,6 +293,18 @@ void goToSleep(uint32_t sysTicksToSleep) } if (online.serialLogging == true) { + if (incomingBufferSpot > 0) + { + //Write the remainder of the buffer + digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording + serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card + digitalWrite(PIN_STAT_LED, LOW); + + incomingBufferSpot = 0; + + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + } + serialDataFile.sync(); updateDataFileAccess(&serialDataFile); // Update the file access time & date serialDataFile.close(); @@ -589,6 +613,18 @@ void stopLogging(void) } if (online.serialLogging == true) { + if (incomingBufferSpot > 0) + { + //Write the remainder of the buffer + digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording + serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card + digitalWrite(PIN_STAT_LED, LOW); + + incomingBufferSpot = 0; + + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + } + serialDataFile.sync(); updateDataFileAccess(&serialDataFile); // Update the file access time & date serialDataFile.close(); diff --git a/Firmware/OpenLog_Artemis/menuMain.ino b/Firmware/OpenLog_Artemis/menuMain.ino index fc38843..9c67548 100644 --- a/Firmware/OpenLog_Artemis/menuMain.ino +++ b/Firmware/OpenLog_Artemis/menuMain.ino @@ -98,6 +98,18 @@ void menuMain(bool alwaysOpen) } if (online.serialLogging == true) { + if (incomingBufferSpot > 0) + { + //Write the remainder of the buffer + digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording + serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card + digitalWrite(PIN_STAT_LED, LOW); + + incomingBufferSpot = 0; + + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + } + serialDataFile.sync(); updateDataFileAccess(&serialDataFile); // Update the file access time & date serialDataFile.close(); @@ -164,6 +176,18 @@ void menuMain(bool alwaysOpen) } if (online.serialLogging == true) { + if (incomingBufferSpot > 0) + { + //Write the remainder of the buffer + digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording + serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card + digitalWrite(PIN_STAT_LED, LOW); + + incomingBufferSpot = 0; + + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + } + serialDataFile.sync(); updateDataFileAccess(&serialDataFile); // Update the file access time & date serialDataFile.close(); diff --git a/Firmware/OpenLog_Artemis/menuSerialLogging.ino b/Firmware/OpenLog_Artemis/menuSerialLogging.ino index 65cc64d..00e0bd9 100644 --- a/Firmware/OpenLog_Artemis/menuSerialLogging.ino +++ b/Firmware/OpenLog_Artemis/menuSerialLogging.ino @@ -81,6 +81,18 @@ void menuSerialLogging() { if (online.serialLogging) { + if (incomingBufferSpot > 0) + { + //Write the remainder of the buffer + digitalWrite(PIN_STAT_LED, HIGH); //Toggle stat LED to indicating log recording + serialDataFile.write(incomingBuffer, incomingBufferSpot); //Record the buffer to the card + digitalWrite(PIN_STAT_LED, LOW); + + incomingBufferSpot = 0; + + lastSeriaLogSyncTime = rtcMillis(); //Reset the last sync time to now + } + //Shut it all down updateDataFileAccess(&serialDataFile); // Update the file access time & date serialDataFile.close(); From c3858a789cedd1934ab6a424b1f310d8d0014d45 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 6 Feb 2025 13:37:20 +0000 Subject: [PATCH 4/7] Add DEL * / RM * to resolve #211 --- Firmware/OpenLog_Artemis/zmodem.ino | 38 ++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/Firmware/OpenLog_Artemis/zmodem.ino b/Firmware/OpenLog_Artemis/zmodem.ino index 6e517d4..1cdc102 100644 --- a/Firmware/OpenLog_Artemis/zmodem.ino +++ b/Firmware/OpenLog_Artemis/zmodem.ino @@ -300,7 +300,43 @@ void sdCardMenu(int numberOfSeconds) else if (!strcmp_P(cmd, PSTR("DEL")) || !strcmp_P(cmd, PSTR("RM"))) // ReMove / DELete file { - if (!sd.remove(param)) + if (!strcmp_P(param, PSTR("*"))) + { + + count_files(&Filesleft, &Totalleft); + DSERIALprint(F("\r\nDeleting ")); DSERIAL->print(Filesleft); DSERIALprint(F(" files (")); DSERIAL->print(Totalleft); DSERIALprintln(F(" bytes)")); + + root.open("/"); // (re)open the root directory + root.rewind(); // rewind + + if (Filesleft > 0) + { + //while (sd.vwd()->readDir(dir) == sizeof(*dir)) { + while (fout.openNext(&root, O_RDONLY)) + { + // read next directory entry in current working directory + if (!fout.isDir()) { + char fname[30]; + size_t fsize = 30; + fout.getName(fname, fsize); + fout.close(); + sd.remove(fname); + } + else + { + fout.close(); + } + } + DSERIALprintln(F("\r\nFiles deleted!\r\n")); + } + else + { + DSERIALprintln(F("\r\nNo files to delete\r\n")); + } + + root.close(); + } + else if (!sd.remove(param)) { DSERIALprint(F("\r\nFailed to delete file ")); DSERIAL->flush(); DSERIAL->println(param); DSERIAL->flush(); From 7311c161ed651e4bc6b250d99d7e72cb143d9f2f Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 6 Feb 2025 14:18:36 +0000 Subject: [PATCH 5/7] Don't delete OLA_ settings.txt --- Firmware/OpenLog_Artemis/zmodem.ino | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Firmware/OpenLog_Artemis/zmodem.ino b/Firmware/OpenLog_Artemis/zmodem.ino index 1cdc102..82028cb 100644 --- a/Firmware/OpenLog_Artemis/zmodem.ino +++ b/Firmware/OpenLog_Artemis/zmodem.ino @@ -320,7 +320,8 @@ void sdCardMenu(int numberOfSeconds) size_t fsize = 30; fout.getName(fname, fsize); fout.close(); - sd.remove(fname); + if ((str_cmp_P(fname, PSTR("OLA_settings.txt"))) && (str_cmp_P(fname, PSTR("OLA_deviceSettings.txt")))) + sd.remove(fname); } else { From faae138a212eee8185d744df5b793c22dde92ef0 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 6 Feb 2025 14:25:49 +0000 Subject: [PATCH 6/7] Fix that typo! --- Firmware/OpenLog_Artemis/zmodem.ino | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Firmware/OpenLog_Artemis/zmodem.ino b/Firmware/OpenLog_Artemis/zmodem.ino index 82028cb..f2e5fbe 100644 --- a/Firmware/OpenLog_Artemis/zmodem.ino +++ b/Firmware/OpenLog_Artemis/zmodem.ino @@ -311,7 +311,6 @@ void sdCardMenu(int numberOfSeconds) if (Filesleft > 0) { - //while (sd.vwd()->readDir(dir) == sizeof(*dir)) { while (fout.openNext(&root, O_RDONLY)) { // read next directory entry in current working directory @@ -320,7 +319,7 @@ void sdCardMenu(int numberOfSeconds) size_t fsize = 30; fout.getName(fname, fsize); fout.close(); - if ((str_cmp_P(fname, PSTR("OLA_settings.txt"))) && (str_cmp_P(fname, PSTR("OLA_deviceSettings.txt")))) + if ((strcmp_P(fname, PSTR("OLA_settings.txt"))) && (strcmp_P(fname, PSTR("OLA_deviceSettings.txt")))) sd.remove(fname); } else From facd024da45769f295df071b0d061866bdb975d0 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 6 Feb 2025 14:49:04 +0000 Subject: [PATCH 7/7] Update sdCardHelp --- Firmware/OpenLog_Artemis/zmodem.ino | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Firmware/OpenLog_Artemis/zmodem.ino b/Firmware/OpenLog_Artemis/zmodem.ino index f2e5fbe..c36b7c6 100644 --- a/Firmware/OpenLog_Artemis/zmodem.ino +++ b/Firmware/OpenLog_Artemis/zmodem.ino @@ -171,7 +171,7 @@ void sdCardHelp(void) DSERIALprintln(F("Available Commands:")); DSERIAL->flush(); DSERIALprintln(F("HELP - Print this list of commands")); DSERIAL->flush(); DSERIALprintln(F("DIR - List files in current working directory - alternate LS")); DSERIAL->flush(); - DSERIALprintln(F("DEL file - Delete file - alternate RM")); DSERIAL->flush(); + DSERIALprintln(F("DEL file - Delete file - alternate RM (\"DEL *\" will delete all files)")); DSERIAL->flush(); DSERIALprintln(F("SZ file - Send file from OLA to terminal using ZModem (\"SZ *\" will send all files)")); DSERIAL->flush(); DSERIALprintln(F("SS file - Send file from OLA using serial TX pin")); DSERIAL->flush(); DSERIALprintln(F("CAT file - Type file to this terminal - alternate TYPE")); DSERIAL->flush(); @@ -304,7 +304,7 @@ void sdCardMenu(int numberOfSeconds) { count_files(&Filesleft, &Totalleft); - DSERIALprint(F("\r\nDeleting ")); DSERIAL->print(Filesleft); DSERIALprint(F(" files (")); DSERIAL->print(Totalleft); DSERIALprintln(F(" bytes)")); + DSERIALprint(F("\r\nDeleting at most ")); DSERIAL->print(Filesleft); DSERIALprint(F(" files (")); DSERIAL->print(Totalleft); DSERIALprintln(F(" bytes)")); root.open("/"); // (re)open the root directory root.rewind(); // rewind