Fix TemplateStream

Tested against large (1MB) JSON listing generated from templates, issues arising.

Need to check for remaining plain data before continuing with next tag search
Move check for missing end tag after `evaluate`
Use a smaller buffer for testing to stress code more

Sming/Core/Data/Stream/TemplateStream.cpp

@@ -52,6 +52,11 @@ uint16_t TemplateStream::readMemoryBlock(char* data, int bufSize)
 		return sendValue();
 	}
 
+	if(valueWaitSize != 0) {
+		size_t res = std::min(uint16_t(bufSize), valueWaitSize);
+		return stream->readMemoryBlock(data, res);
+	}
+
 	const size_t tagDelimiterLength = 1 + doubleBraces;
 
 	if(size_t(bufSize) <= TEMPLATE_MAX_VAR_NAME_LEN + (2 * tagDelimiterLength)) {
@@ -68,12 +73,23 @@ uint16_t TemplateStream::readMemoryBlock(char* data, int bufSize)
 	if(datalen != 0) {
 		data[datalen] = '\0'; // Terminate buffer to mitigate overflow risk
 		auto tagStart = findStartTag(data);
-		auto lastTagFound = tagStart;
 		while(tagStart != nullptr) {
-			lastTagFound = tagStart;
-
 			char* curPos = tagStart + tagDelimiterLength;
 			value = evaluate(curPos);
+			if(size_t(curPos - data) > datalen) {
+				// Incomplete variable name, end tag not found in buffer
+				unsigned newlen = tagStart - data;
+				if(newlen + TEMPLATE_MAX_VAR_NAME_LEN > datalen) {
+					// Return what we have so far, unless we're at the end of the input stream
+					if(datalen == size_t(bufSize - 1)) {
+						debug_d("TemplateStream: trim end to %u from %u", newlen, datalen);
+						datalen = newlen;
+						break;
+					}
+				}
+				// debug_d("[TMPL] TMVNL too small");
+				// datalen = lastTagFound - data;
+			}
 			if(doubleBraces) {
 				// Double end brace isn't necessary, but if present skip it
 				if(*curPos == '}') {
@@ -113,18 +129,6 @@ uint16_t TemplateStream::readMemoryBlock(char* data, int bufSize)
 			memmove(data, start, valueWaitSize);
 			return valueWaitSize;
 		}
-
-		if(lastTagFound != nullptr) {
-			unsigned newlen = lastTagFound - data;
-			if(newlen + TEMPLATE_MAX_VAR_NAME_LEN > datalen) {
-				debug_d("TemplateStream: trim end to %u from %u", newlen, datalen);
-				// It can be a incomplete variable name - don't split it
-				// provided we're not at end of input stream
-				if(datalen == size_t(bufSize)) {
-					datalen = newlen;
-				}
-			}
-		}
 	}
 
 	datalen -= (start - data);

tests/HostTests/modules/TemplateStream.cpp

@@ -117,7 +117,7 @@ class TemplateStreamTest : public TestGroup
 private:
 	void check(TemplateStream& tmpl, const FlashString& ref)
 	{
-		static constexpr size_t bufSize{512};
+		static constexpr size_t bufSize{220};
 		char buf1[bufSize];
 		char buf2[bufSize];
 		FSTR::Stream refStream(ref);