From ca5a010fc7a77877cad24277c858dfb455c8733d Mon Sep 17 00:00:00 2001 From: Schmurtz Date: Tue, 22 Feb 2022 00:27:57 +0100 Subject: [PATCH] Adding support for Tranfer-Encoding: chunked streams Included changes from yoav-klein and DeqingSun to manage correctly Google TTS. See here for details : https://github.com/earlephilhower/ESP8266Audio/pull/394 --- src/AudioFileSourceHTTPStream.cpp | 119 +++++++++++++++- src/AudioFileSourceHTTPStream.h | 16 ++- src/AudioFileSourceICYStream.cpp | 226 +++++++++++++++--------------- src/AudioFileSourceICYStream.h | 8 +- 4 files changed, 249 insertions(+), 120 deletions(-) diff --git a/src/AudioFileSourceHTTPStream.cpp b/src/AudioFileSourceHTTPStream.cpp index c5d0b833..d8456ea2 100644 --- a/src/AudioFileSourceHTTPStream.cpp +++ b/src/AudioFileSourceHTTPStream.cpp @@ -27,13 +27,50 @@ AudioFileSourceHTTPStream::AudioFileSourceHTTPStream() pos = 0; reconnectTries = 0; saveURL[0] = 0; + next_chunk = 0; + eof = false; } AudioFileSourceHTTPStream::AudioFileSourceHTTPStream(const char *url) { saveURL[0] = 0; reconnectTries = 0; + next_chunk = 0; open(url); + +} + +bool AudioFileSourceHTTPStream::verifyCrlf() +{ + + uint8_t crlf[3]; + + client.read(crlf, 2); + crlf[2] = 0; + + return !strncmp("\r\n", reinterpret_cast(crlf), 2); +} + +int AudioFileSourceHTTPStream::getChunkSize() +{ + unsigned long start = millis(); + while ((client.available() == 0) && (((signed long)(millis() - start)) < 1500)){ + yield(); + } + if (client.available() == 0) return -1; + String length = client.readStringUntil('\r'); + String lf = client.readStringUntil('\n'); + + unsigned int val = 0; + auto ret = sscanf(length.c_str(), "%x", &val); + if(ret) + { + return val; + } + else + { + return -1; + } } bool AudioFileSourceHTTPStream::open(const char *url) @@ -44,12 +81,37 @@ bool AudioFileSourceHTTPStream::open(const char *url) #ifndef ESP32 http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); #endif + const char* headers[] = { "Transfer-Encoding" }; + http.collectHeaders( headers, 1 ); int code = http.GET(); if (code != HTTP_CODE_OK) { http.end(); cb.st(STATUS_HTTPFAIL, PSTR("Can't open HTTP request")); return false; } + if (http.hasHeader("Transfer-Encoding")) { + audioLogger->printf_P(PSTR("Transfer-Encoding: %s\n"), http.header("Transfer-Encoding").c_str()); + if(http.header("Transfer-Encoding") == String(PSTR("chunked"))) { + + next_chunk = getChunkSize(); + if(-1 == next_chunk) + { + return false; + } + is_chunked = true; + readImpl = &AudioFileSourceHTTPStream::readChunked; + } else { + is_chunked = false; + readImpl = &AudioFileSourceHTTPStream::readRegular; + } + + } else { + readImpl = &AudioFileSourceHTTPStream::readRegular; + audioLogger->printf_P(PSTR("No Transfer-Encoding\n")); + is_chunked = false; + } + + size = http.getSize(); strncpy(saveURL, url, sizeof(saveURL)); saveURL[sizeof(saveURL)-1] = 0; @@ -61,13 +123,60 @@ AudioFileSourceHTTPStream::~AudioFileSourceHTTPStream() http.end(); } +uint32_t AudioFileSourceHTTPStream::readRegular(void *data, uint32_t len, bool nonBlock) +{ + return readInternal(data, len, nonBlock); +} + +uint32_t AudioFileSourceHTTPStream::readChunked(void *data, uint32_t len, bool nonBlock) +{ + uint32_t bytesRead = 0; + uint32_t pos = 0; + + if(len > 0) + { + if(len >= next_chunk) + { + if (next_chunk) + { + bytesRead = readInternal((void*)(((uint8_t*)data) + pos), next_chunk, nonBlock); + next_chunk -= bytesRead; + pos += bytesRead; + } + len -= pos; + if (!next_chunk){ + if(!verifyCrlf()) + { + audioLogger->printf(PSTR("Couldn't read CRLF after chunk, something is wrong !!\n")); + return 0; + } + next_chunk = getChunkSize(); + if (next_chunk < 0){ + //timeout EOF + close(); + } + } + } + else + { + bytesRead = readInternal((void*)(((uint8_t*)data) + pos), len, nonBlock); + next_chunk -= bytesRead; + len -= bytesRead; + pos += bytesRead; + } + } + return pos; +} + uint32_t AudioFileSourceHTTPStream::read(void *data, uint32_t len) { if (data==NULL) { audioLogger->printf_P(PSTR("ERROR! AudioFileSourceHTTPStream::read passed NULL data\n")); return 0; } - return readInternal(data, len, false); + + return (this->*readImpl)(data, len, false); + } uint32_t AudioFileSourceHTTPStream::readNonBlock(void *data, uint32_t len) @@ -76,7 +185,8 @@ uint32_t AudioFileSourceHTTPStream::readNonBlock(void *data, uint32_t len) audioLogger->printf_P(PSTR("ERROR! AudioFileSourceHTTPStream::readNonBlock passed NULL data\n")); return 0; } - return readInternal(data, len, true); + return (this->*readImpl)(data, len, true); + } uint32_t AudioFileSourceHTTPStream::readInternal(void *data, uint32_t len, bool nonBlock) @@ -115,7 +225,7 @@ uint32_t AudioFileSourceHTTPStream::readInternal(void *data, uint32_t len, bool size_t avail = stream->available(); if (!nonBlock && !avail) { cb.st(STATUS_NODATA, PSTR("No stream data available")); - http.end(); + close(); goto retry; } if (avail == 0) return 0; @@ -137,12 +247,13 @@ bool AudioFileSourceHTTPStream::seek(int32_t pos, int dir) bool AudioFileSourceHTTPStream::close() { http.end(); + eof = true; return true; } bool AudioFileSourceHTTPStream::isOpen() { - return http.connected(); + return http.connected() && (!eof); } uint32_t AudioFileSourceHTTPStream::getSize() diff --git a/src/AudioFileSourceHTTPStream.h b/src/AudioFileSourceHTTPStream.h index 34e54663..2baf6918 100644 --- a/src/AudioFileSourceHTTPStream.h +++ b/src/AudioFileSourceHTTPStream.h @@ -21,6 +21,8 @@ #if defined(ESP32) || defined(ESP8266) #pragma once +#include // std::map ascii_to_hex; + #include #ifdef ESP32 #include @@ -29,6 +31,8 @@ #endif #include "AudioFileSource.h" + + class AudioFileSourceHTTPStream : public AudioFileSource { friend class AudioFileSourceICYStream; @@ -52,7 +56,9 @@ class AudioFileSourceHTTPStream : public AudioFileSource enum { STATUS_HTTPFAIL=2, STATUS_DISCONNECTED, STATUS_RECONNECTING, STATUS_RECONNECTED, STATUS_NODATA }; private: - virtual uint32_t readInternal(void *data, uint32_t len, bool nonBlock); + bool is_chunked; + int next_chunk; + bool eof; WiFiClient client; HTTPClient http; int pos; @@ -60,6 +66,14 @@ class AudioFileSourceHTTPStream : public AudioFileSource int reconnectTries; int reconnectDelayMs; char saveURL[128]; + uint32_t (AudioFileSourceHTTPStream::*readImpl)(void *data, uint32_t len, bool nonBlock); + + virtual uint32_t readInternal(void *data, uint32_t len, bool nonBlock); + uint32_t readChunked(void *data, uint32_t len, bool nonBlock); + uint32_t readRegular(void *data, uint32_t len, bool nonBlock); + bool verifyCrlf(); + int getChunkSize(); + }; diff --git a/src/AudioFileSourceICYStream.cpp b/src/AudioFileSourceICYStream.cpp index edeba761..fd4286ff 100644 --- a/src/AudioFileSourceICYStream.cpp +++ b/src/AudioFileSourceICYStream.cpp @@ -20,7 +20,9 @@ #if defined(ESP32) || defined(ESP8266) +#if !defined(_GNU_SOURCE) #define _GNU_SOURCE +#endif #include "AudioFileSourceICYStream.h" #include @@ -41,11 +43,11 @@ AudioFileSourceICYStream::AudioFileSourceICYStream(const char *url) bool AudioFileSourceICYStream::open(const char *url) { - static const char *hdr[] = { "icy-metaint", "icy-name", "icy-genre", "icy-br" }; + static const char *hdr[] = { "icy-metaint", "icy-name", "icy-genre", "icy-br", "Transfer-Encoding" }; pos = 0; http.begin(client, url); http.addHeader("Icy-MetaData", "1"); - http.collectHeaders( hdr, 4 ); + http.collectHeaders( hdr, 5 ); http.setReuse(true); http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); int code = http.GET(); @@ -72,8 +74,32 @@ bool AudioFileSourceICYStream::open(const char *url) String ret = http.header(hdr[3]); // cb.md("Bitrate", false, ret.c_str()); } + if (http.hasHeader(hdr[4])) { + audioLogger->printf_P(PSTR("Transfer-Encoding: %s\n"), http.header("Transfer-Encoding").c_str()); + if(http.header("Transfer-Encoding") == String(PSTR("chunked"))) { + + next_chunk = getChunkSize(); + if(-1 == next_chunk) + { + return false; + } + is_chunked = true; + readImpl = &AudioFileSourceHTTPStream::readChunked; + } else { + is_chunked = false; + readImpl = &AudioFileSourceHTTPStream::readRegular; + } + + } else { + readImpl = &AudioFileSourceHTTPStream::readRegular; + audioLogger->printf_P(PSTR("No Transfer-Encoding\n")); + is_chunked = false; + } icyByteCount = 0; + mdSize = 0; + readingIcy = false; + memset(icyBuff, 0, sizeof(icyBuff)); size = http.getSize(); strncpy(saveURL, url, sizeof(saveURL)); saveURL[sizeof(saveURL)-1] = 0; @@ -85,138 +111,110 @@ AudioFileSourceICYStream::~AudioFileSourceICYStream() http.end(); } -uint32_t AudioFileSourceICYStream::readInternal(void *data, uint32_t len, bool nonBlock) +uint32_t AudioFileSourceICYStream::read(void *data, uint32_t len) { - // Ensure we can't possibly read 2 ICY headers in a single go #355 - if (icyMetaInt > 1) { - len = std::min((int)(icyMetaInt >> 1), (int)len); - } -retry: - if (!http.connected()) { - cb.st(STATUS_DISCONNECTED, PSTR("Stream disconnected")); - http.end(); - for (int i = 0; i < reconnectTries; i++) { - char buff[64]; - sprintf_P(buff, PSTR("Attempting to reconnect, try %d"), i); - cb.st(STATUS_RECONNECTING, buff); - delay(reconnectDelayMs); - if (open(saveURL)) { - cb.st(STATUS_RECONNECTED, PSTR("Stream reconnected")); - break; - } - } - if (!http.connected()) { - cb.st(STATUS_DISCONNECTED, PSTR("Unable to reconnect")); - return 0; - } + if (data==NULL) { + audioLogger->printf_P(PSTR("ERROR! AudioFileSourceHTTPStream::read passed NULL data\n")); + return 0; } - if ((size > 0) && (pos >= size)) return 0; - - WiFiClient *stream = http.getStreamPtr(); - - // Can't read past EOF... - if ( (size > 0) && (len > (uint32_t)(pos - size)) ) len = pos - size; - - if (!nonBlock) { - int start = millis(); - while ((stream->available() < (int)len) && (millis() - start < 500)) yield(); + + uint32_t readLen = 0; + uint32_t toRead = len; + unsigned long start = millis(); + while ((readLen < len) && (((signed long)(millis() - start)) < 1500)){ + if (!isOpen()) break; + uint32_t ret = readWithIcy((void*)(((uint8_t*)data) + readLen), toRead, true); + readLen += ret; + toRead -= ret; } + + return readLen; +} - size_t avail = stream->available(); - if (!nonBlock && !avail) { - cb.st(STATUS_NODATA, PSTR("No stream data available")); - http.end(); - goto retry; +uint32_t AudioFileSourceICYStream::readNonBlock(void *data, uint32_t len) +{ + if (data==NULL) { + audioLogger->printf_P(PSTR("ERROR! AudioFileSourceHTTPStream::readNonBlock passed NULL data\n")); + return 0; } - if (avail == 0) return 0; - if (avail < len) len = avail; + + uint32_t readLen = readWithIcy(data, len, true); + + return readLen; +} +uint32_t AudioFileSourceICYStream::readWithIcy(void *data, uint32_t len, bool nonBlock){ int read = 0; int ret = 0; - // If the read would hit an ICY block, split it up... - if (((int)(icyByteCount + len) > (int)icyMetaInt) && (icyMetaInt > 0)) { - int beforeIcy = icyMetaInt - icyByteCount; + + if (readingIcy){ + if (mdSize>0){ + uint32_t readMdLen = std::min((int)(mdSize), (int)len); + readMdLen = std::min((int)(readMdLen), (int)256); //the buffer is only 256 big + ret = (this->*readImpl)(data, readMdLen, nonBlock); + if (ret < 0) ret = 0; + len -= ret; + mdSize -= ret; + + if (ret > 0){ + char *readInto = icyBuff + 256; + memcpy(readInto, data, ret); + int end = 256 + ret; // The last byte of valid data + char *header = (char *)memmem((void*)icyBuff, end, (void*)"StreamTitle=", 12); + if (header) { + char *p = header+12; + if (*p=='\'' || *p== '"' ) { + char closing[] = { *p, ';', '\0' }; + char *psz = strstr( p+1, closing ); + if( !psz ) psz = strchr( p+1, ';' ); + if( psz ) *psz = '\0'; + p++; + } else { + char *psz = strchr( p, ';' ); + if( psz ) *psz = '\0'; + } + cb.md("StreamTitle", false, p); + } + memmove(icyBuff, icyBuff+ret, 256); + + } + if (ret != readMdLen) return read; // Partial read + } + if (mdSize<=0){ + icyByteCount = 0; + mdSize = 0; + readingIcy = false; + } + }else{ + int beforeIcy; + if (((int)(icyByteCount + len) >= (int)icyMetaInt) && (icyMetaInt > 0)) { + beforeIcy = icyMetaInt - icyByteCount; + }else{ + beforeIcy = len; + } if (beforeIcy > 0) { - ret = stream->read(reinterpret_cast(data), beforeIcy); + ret = (this->*readImpl)(data, beforeIcy, nonBlock); if (ret < 0) ret = 0; read += ret; - pos += ret; len -= ret; data = (void *)(reinterpret_cast(data) + ret); icyByteCount += ret; if (ret != beforeIcy) return read; // Partial read } - - // ICY MD handling - int mdSize; - uint8_t c; - int mdret = stream->read(&c, 1); - if (mdret==0) return read; - mdSize = c * 16; - if ((mdret == 1) && (mdSize > 0)) { - // This is going to get ugly fast. - char icyBuff[256 + 16 + 1]; - char *readInto = icyBuff + 16; - memset(icyBuff, 0, 16); // Ensure no residual matches occur - while (mdSize) { - int toRead = mdSize > 256 ? 256 : mdSize; - int ret = stream->read((uint8_t*)readInto, toRead); - if (ret < 0) return read; - if (ret == 0) { delay(1); continue; } - mdSize -= ret; - // At this point we have 0...15 = last 15 chars read from prior read plus new data - int end = 16 + ret; // The last byte of valid data - char *header = (char *)memmem((void*)icyBuff, end, (void*)"StreamTitle=", 12); - if (!header) { - // No match, so move the last 16 bytes back to the start and continue - memmove(icyBuff, icyBuff+end-16, 16); - delay(1); - continue; - } - // Found header, now move it to the front - int lastValidByte = end - (header -icyBuff) + 1; - memmove(icyBuff, header, lastValidByte); - // Now fill the buffer to the end with read data - while (mdSize && lastValidByte < 255) { - int toRead = mdSize > (256 - lastValidByte) ? (256 - lastValidByte) : mdSize; - ret = stream->read((uint8_t*)icyBuff + lastValidByte, toRead); - if (ret==-1) return read; // error - if (ret == 0) { delay(1); continue; } - mdSize -= ret; - lastValidByte += ret; - } - // Buffer now contains StreamTitle=....., parse it - char *p = icyBuff+12; - if (*p=='\'' || *p== '"' ) { - char closing[] = { *p, ';', '\0' }; - char *psz = strstr( p+1, closing ); - if( !psz ) psz = strchr( &icyBuff[13], ';' ); - if( psz ) *psz = '\0'; - p++; - } else { - char *psz = strchr( p, ';' ); - if( psz ) *psz = '\0'; - } - cb.md("StreamTitle", false, p); - - // Now skip rest of MD block - while (mdSize) { - int toRead = mdSize > 256 ? 256 : mdSize; - ret = stream->read((uint8_t*)icyBuff, toRead); - if (ret < 0) return read; - if (ret == 0) { delay(1); continue; } - mdSize -= ret; - } + if (len >= 1 ){ //try to read mdSize + ret = (this->*readImpl)(data, 1, nonBlock); + if (ret < 0) ret = 0; + if (ret==1){ + mdSize = *((uint8_t*)data) * 16; + readingIcy = true; + memset(icyBuff, 0, 256); // Ensure no residual matches occur } + len -= ret; + data = (void *)(reinterpret_cast(data) + ret); + if (ret != 1) return read; // Partial read } - icyByteCount = 0; } - ret = stream->read(reinterpret_cast(data), len); - if (ret < 0) ret = 0; - read += ret; - pos += ret; - icyByteCount += ret; return read; } diff --git a/src/AudioFileSourceICYStream.h b/src/AudioFileSourceICYStream.h index 97688a57..badcf911 100644 --- a/src/AudioFileSourceICYStream.h +++ b/src/AudioFileSourceICYStream.h @@ -38,11 +38,17 @@ class AudioFileSourceICYStream : public AudioFileSourceHTTPStream virtual ~AudioFileSourceICYStream() override; virtual bool open(const char *url) override; + virtual uint32_t read(void *data, uint32_t len) override; + virtual uint32_t readNonBlock(void *data, uint32_t len) override; + private: - virtual uint32_t readInternal(void *data, uint32_t len, bool nonBlock) override; + uint32_t readWithIcy(void *data, uint32_t len, bool nonBlock); int icyMetaInt; int icyByteCount; + int mdSize; + bool readingIcy; + char icyBuff[256 + 256 + 1]; //256 for new data, 256 for old data }; #endif