Add streaming filesystem backup support (#2323)

This PR adds the `ArchiveStream` class to support backing up filesystems. The archive files are in FWFS format.

- Add FWFS for user attribute support

User attributes can be set (via callback) during the backup operation and read backup using the standard IFS API.

- Add Encrypted file attribute

Files may be encrypted via `IBlockEncoder` plugin, which would set this attribute.

- Add `Comment` standard attribute

Standard comment of up to 255 characters (FWFS and LittleFS).

- Add check in fsbuild for duplicate directories/files, and allow paths to be specified in source

Tests have been updated accordingly.

Sming/Core/Data/Stream/IFS/ArchiveStream.h

@@ -0,0 +1,28 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/anakod/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * ArchiveStream.h
+ *
+ * @author mikee47 <mike@sillyhouse.net> April 2021
+ *
+ *
+ ****/
+
+#pragma once
+
+#include <IFS/FWFS/ArchiveStream.h>
+#include <FileSystem.h>
+
+class ArchiveStream : public IFS::FWFS::ArchiveStream
+{
+public:
+	using IFS::FWFS::ArchiveStream::ArchiveStream;
+
+	ArchiveStream(VolumeInfo volumeInfo, String rootPath = nullptr, Flags flags = 0)
+		: ArchiveStream(::getFileSystem(), volumeInfo, rootPath, flags)
+	{
+	}
+};

Sming/Core/Data/Stream/MemoryDataStream.h

@@ -104,6 +104,16 @@ class MemoryDataStream : public ReadWriteStream
 		readPos = 0;
 	}
 
+	size_t getSize() const
+	{
+		return size;
+	}
+
+	size_t getCapacity() const
+	{
+		return capacity;
+	}
+
 private:
 	char* buffer = nullptr;			///< Stream content stored here
 	size_t maxCapacity{UINT16_MAX}; ///< Limit size of stream

samples/Basic_IFS/README.rst

@@ -7,7 +7,14 @@ Simple Webserver demonstration using IFS.
 
 View the filesystem using a web browser.
 
-To see the data in a different format, add ``?format=XX``. Current supported formats are ``json``, ``text`` and ``html``.
+To see directory content in a different format, append ``?format=XX``, one of:
+
+- ``json``
+- ``text``
+- ``html``
+
+Use the format ``archive`` to retrieve an archive/backup of the directory tree as an FWFS image.
+
 
 Building
 --------

samples/Basic_IFS/app/application.cpp

@@ -8,6 +8,7 @@
 #include <Data/Stream/IFS/DirectoryTemplate.h>
 #include <Data/Stream/IFS/HtmlDirectoryTemplate.h>
 #include <Data/Stream/IFS/JsonDirectoryTemplate.h>
+#include <Data/Stream/IFS/ArchiveStream.h>
 #include <Storage/ProgMem.h>
 #include <LittleFS.h>
 
@@ -52,11 +53,25 @@ void onFile(HttpRequest& request, HttpResponse& response)
 	++requestCount;
 
 	String file = request.uri.getRelativePath();
+	String fmt = request.uri.Query["format"];
 
 	if(dirExist(file)) {
+		if(fmt.equalsIgnoreCase("archive")) {
+			debug_i("Sending streaming archive");
+			IFS::FileSystem::NameInfo fsinfo;
+			fileGetSystemInfo(fsinfo);
+			ArchiveStream::VolumeInfo volumeInfo;
+			volumeInfo.name = F("Backup of '") + fsinfo.name + "'";
+			if(file.length() != 0) {
+				volumeInfo.name += F("; root = '") + file + "'";
+			}
+			auto archive = new ArchiveStream(volumeInfo, file, ArchiveStream::Flag::IncludeMountPoints);
+			response.sendDataStream(archive, archive->getMimeType());
+			return;
+		}
+
 		auto dir = new Directory;
 		IFS::DirectoryTemplate* tmpl;
-		String fmt = request.uri.Query["format"];
 		if(fmt.equalsIgnoreCase("json")) {
 			auto source = new FlashMemoryStream(listing_json);
 			tmpl = new IFS::JsonDirectoryTemplate(source, dir);
@@ -71,24 +86,32 @@ void onFile(HttpRequest& request, HttpResponse& response)
 		dir->open(file);
 		tmpl->gotoSection(0);
 		response.sendDataStream(tmpl, tmpl->getMimeType());
-	} else {
-		//	response.setCache(86400, true); // It's important to use cache for better performance.
-		auto stream = new FileStream;
-		if(!stream->open(file)) {
-			response.code = HTTP_STATUS_INTERNAL_SERVER_ERROR;
-			delete stream;
-			return;
-		}
-		FileStat stat;
-		stream->stat(stat);
-		if(stat.compression.type == IFS::Compression::Type::GZip) {
-			response.headers[HTTP_HEADER_CONTENT_ENCODING] = F("gzip");
-		} else if(stat.compression.type != IFS::Compression::Type::None) {
-			debug_e("Unsupported compression type: %u", stat.compression.type);
-		}
-		auto mimeType = ContentType::fromFullFileName(file, MIME_TEXT);
-		response.sendDataStream(stream, mimeType);
+		return;
+	}
+
+	if(fmt) {
+		debug_e("'format' option only supported for directories");
+		response.code = HTTP_STATUS_BAD_REQUEST;
+		return;
+	}
+
+	//	response.setCache(86400, true); // It's important to use cache for better performance.
+	auto stream = new FileStream;
+	if(!stream->open(file)) {
+		int err = stream->getLastError();
+		response.code = (err == IFS::Error::NotFound) ? HTTP_STATUS_NOT_FOUND : HTTP_STATUS_INTERNAL_SERVER_ERROR;
+		delete stream;
+		return;
+	}
+	FileStat stat;
+	stream->stat(stat);
+	if(stat.compression.type == IFS::Compression::Type::GZip) {
+		response.headers[HTTP_HEADER_CONTENT_ENCODING] = F("gzip");
+	} else if(stat.compression.type != IFS::Compression::Type::None) {
+		debug_e("Unsupported compression type: %u", stat.compression.type);
 	}
+	auto mimeType = ContentType::fromFullFileName(file, MIME_TEXT);
+	response.sendDataStream(stream, mimeType);
 }
 
 void startWebServer()