Add dataset templating support (#2164)

This PR extends templated streaming to support formatting of large datasets.
It is used by #1579 to provide a mechanism for streaming directory listings in various formats.

Upgrade TemplateStream

* Malformed template data can cause lockup waiting for next input when there isn't any.
* Add output enable control to support filtering
* Allow spaces in tag names, increase max length to 128
* Add `evaluate` method to enable support for extended tag parsing
* Add optional `owned` parameter to TemplateStream constructor for situations where caller is responsible for allocation
* Allow TemplateStream to be rewound: implement `seekFrom`
* Add option to use double braces {{ }} - necessary for JSON

Add class-based support for escaping/formatting strings (html, json, etc)

* Base Formatter class, headers and standard/html/json within Format namespace

Add SectionStream class

* Presents each marked section within a source stream as a separate stream
* Sections are identified by index and can be selected to allow repeating of sections, etc. as required

Add SectionTemplate class to support dataset reporting

* Adds basic expression parsing with conditionals, built-in functions, etc.
* Uses SectionStream to support typical reports with a header/content/footer. The content section is repeated for each record.

Add separate TemplateStream test module, with more tests

Sming/Core/Data/Format.h

@@ -0,0 +1,21 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * Format.h
+ *
+ * @author mikee47 <mike@sillyhouse.net> Nov 2020
+ *
+ ****/
+
+#pragma once
+
+#include <WString.h>
+
+#include "Format/Standard.h"
+#include "Format/Html.h"
+#include "Format/Json.h"
+
+using Formatter = Format::Formatter;

Sming/Core/Data/Format/Formatter.h

@@ -0,0 +1,48 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * Formatter.h
+ *
+ * @author mikee47 <mike@sillyhouse.net> Nov 2020
+ *
+ ****/
+
+#pragma once
+
+#include <WString.h>
+#include <Network/WebConstants.h>
+
+namespace Format
+{
+/**
+ * @brief Virtual class to perform format-specific String adjustments
+ */
+class Formatter
+{
+public:
+	/**
+	 * @brief Perform any necessary text escaping so output is valid
+	 */
+	virtual void escape(String& value) const = 0;
+
+	/**
+	 * @brief Convert a value into quoted string
+	 */
+	virtual void quote(String& value) const = 0;
+
+	/**
+	 * @brief Remove any quotes from a value
+	 */
+	virtual void unQuote(String& value) const = 0;
+
+	/**
+	 * @brief Correspdoning MIME type for this format
+	 * @note New types must be added to WebConstants.h
+	 */
+	virtual MimeType mimeType() const = 0;
+};
+
+} // namespace Format

Sming/Core/Data/Format/Html.cpp

@@ -0,0 +1,34 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * Html.cpp
+ *
+ * @author mikee47 <mike@sillyhouse.net> May 2019
+ *
+ ****/
+
+#include "Html.h"
+#include <Network/WebHelpers/escape.h>
+
+namespace Format
+{
+Html html;
+
+void Html::escape(String& value) const
+{
+	auto len = html_escape_len(value.c_str(), value.length());
+	if(len == value.length()) {
+		// No escaping required
+		return;
+	}
+
+	String html;
+	html.setLength(len);
+	html_escape(html.begin(), html.length(), value.c_str());
+	value = html;
+}
+
+} // namespace Format

Sming/Core/Data/Format/Html.h

@@ -0,0 +1,32 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * Html.h
+ *
+ * @author mikee47 <mike@sillyhouse.net> Nov 2022
+ *
+ ****/
+
+#pragma once
+
+#include "Standard.h"
+
+namespace Format
+{
+class Html : public Standard
+{
+public:
+	void escape(String& value) const override;
+
+	MimeType mimeType() const override
+	{
+		return MIME_HTML;
+	}
+};
+
+extern Html html;
+
+} // namespace Format

Sming/Core/Data/Format/Json.cpp

@@ -0,0 +1,81 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * JsonDirectoryTemplate.h
+ *
+ * @author mikee47 <mike@sillyhouse.net> Nov 2020
+ *
+ ****/
+
+#include "Json.h"
+#include <debug_progmem.h>
+
+namespace Format
+{
+Json json;
+
+namespace
+{
+bool IsValidUtf8(const char* str, unsigned length)
+{
+	if(str == nullptr) {
+		return true;
+	}
+
+	unsigned i = 0;
+	while(i < length) {
+		char c = str[i++];
+		if((c & 0x80) == 0) {
+			continue;
+		}
+
+		if(i >= length) {
+			return false; // incomplete multibyte char
+		}
+
+		if(c & 0x20) {
+			c = str[i++];
+			if((c & 0xC0) != 0x80) {
+				return false; // malformed trail byte or out of range char
+			}
+			if(i >= length) {
+				return false; // incomplete multibyte char
+			}
+		}
+
+		c = str[i++];
+		if((c & 0xC0) != 0x80) {
+			return false; // malformed trail byte
+		}
+	}
+
+	return true;
+}
+
+} // namespace
+
+/*
+ * Check for invalid characters and replace them - can break browser
+ * operation otherwise.
+ *
+ * This can occur if filenames become corrupted, so here we just
+ * subsitute an underscore _ for anything which fails to match UTF8.
+ *
+ * TODO: Perform ANSI -> UTF8 conversion?
+ */
+void Json::escape(String& value) const
+{
+	if(!IsValidUtf8(value.c_str(), value.length())) {
+		debug_w("Invalid UTF8: %s", value.c_str());
+		for(unsigned i = 0; i < value.length(); ++i) {
+			char& c = value[i];
+			if(c < 0x20 || c > 127)
+				c = '_';
+		}
+	}
+}
+
+} // namespace Format

Sming/Core/Data/Format/Json.h

@@ -0,0 +1,32 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * Json.h
+ *
+ * @author mikee47 <mike@sillyhouse.net> Nov 2020
+ *
+ ****/
+
+#pragma once
+
+#include "Standard.h"
+
+namespace Format
+{
+class Json : public Standard
+{
+public:
+	void escape(String& value) const override;
+
+	MimeType mimeType() const override
+	{
+		return MIME_JSON;
+	}
+};
+
+extern Json json;
+
+} // namespace Format

Sming/Core/Data/Format/Standard.cpp

@@ -0,0 +1,18 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * Standard.cpp
+ *
+ * @author mikee47 <mike@sillyhouse.net> Nov 2020
+ *
+ ****/
+
+#include "Standard.h"
+
+namespace Format
+{
+Standard standard;
+}

Sming/Core/Data/Format/Standard.h

@@ -0,0 +1,56 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * Standard.h
+ *
+ * @author mikee47 <mike@sillyhouse.net> Nov 2020
+ *
+ ****/
+
+#pragma once
+
+#include "Formatter.h"
+
+namespace Format
+{
+class Standard : public Formatter
+{
+public:
+	void escape(String& value) const override
+	{
+	}
+
+	void quote(String& value) const override
+	{
+		char cQuote{'"'};
+		auto len = value.length();
+		value.setLength(len + 2);
+		memmove(&value[1], value.c_str(), len);
+		value[0] = cQuote;
+		value[len + 1] = cQuote;
+	}
+
+	void unQuote(String& value) const override
+	{
+		char quote = value[0];
+		if(quote == '"' || quote == '\'') {
+			auto len = value.length();
+			if(len > 1 && value[len - 1] == quote) {
+				value.remove(len - 1, 1);
+				value.remove(0, 1);
+			}
+		}
+	}
+
+	MimeType mimeType() const override
+	{
+		return MIME_TEXT;
+	}
+};
+
+extern Standard standard;
+
+} // namespace Format