WIP: Initial implementation of persistence

Persistence implemented with the help of the ArduinoJson library using
MessagePack format.

library.json

@@ -0,0 +1,34 @@
+{
+	"name": "reticulum-cpp",
+	"version": "0.1.1",
+	"description": "Port of Reticulum Network Stack to C++ specifically but not exclusively targeting 32-bit and better MCUs",
+	"keywords": "reticulum, mcu, esp32",
+	"repository":
+	{
+		"type": "git",
+		"url": "https://github.com/attermann/reticulum-cpp.git"
+	},
+	"authors":
+	[
+		{
+			"name": "Chad Attermann",
+			"email": "attermann@gmail.com",
+			"maintainer": true
+		}
+	],
+	"license": "Apache-2.0",
+	"homepage": "https://reticulum.network/",
+	"dependencies": {
+		"ArduinoJson": "~6.21.3",
+		"rweather/Crypto": "~0.4.0",
+		"sandeepmistry/LoRa": "~0.8.0",
+		"WiFi": "~2.0.0"
+	},
+	"frameworks": "*",
+	"platforms": "*",
+	"export": {},
+	"exclude": [
+		".github",
+		"src/main.cpp"
+	]
+}

platformio.ini

@@ -23,6 +23,7 @@ build_flags =
 	-Isrc
 	-DNATIVE
 lib_deps = 
+	ArduinoJson@^6.21.3
 	rweather/Crypto@^0.4.0
 lib_compat_mode = off
 
@@ -54,6 +55,7 @@ build_flags =
 	-Wno-format
 	-Isrc
 lib_deps = 
+	ArduinoJson@^6.21.3
 	rweather/Crypto@^0.4.0
 	sandeepmistry/LoRa@^0.8.0
 	WiFi@^2.0.0

src/Bytes.h

@@ -2,6 +2,8 @@
 
 #include "Log.h"
 
+#include <ArduinoJson.h>
+
 #include <stdint.h>
 #include <stddef.h>
 #include <string.h>
@@ -257,3 +259,15 @@ inline RNS::Bytes& operator << (RNS::Bytes& lhbytes, uint8_t rhbyte) {
 	lhbytes.append(rhbyte);
 	return lhbytes;
 }
+
+namespace ArduinoJson {
+	inline bool convertToJson(const RNS::Bytes& src, JsonVariant dst) {
+		return dst.set(src.toHex());
+	}
+	inline void convertFromJson(JsonVariantConst src, RNS::Bytes& dst) {
+		dst.assignHex(src.as<const char*>());
+	}
+	inline bool canConvertFromJson(JsonVariantConst src, const RNS::Bytes&) {
+		return src.is<const char*>();
+	}
+}

src/Identity.cpp

@@ -36,16 +36,16 @@ void Identity::createKeys() {
 	_object->_prv_bytes     = _object->_prv->private_bytes();
 	debug("Identity::createKeys: prv bytes:     " + _object->_prv_bytes.toHex());
 
-	// CRYPTO: create encryption public keys
-	_object->_pub           = _object->_prv->public_key();
-	_object->_pub_bytes     = _object->_pub->public_bytes();
-	debug("Identity::createKeys: pub bytes:     " + _object->_pub_bytes.toHex());
-
 	// CRYPTO: create signature private keys
 	_object->_sig_prv       = Cryptography::Ed25519PrivateKey::generate();
 	_object->_sig_prv_bytes = _object->_sig_prv->private_bytes();
 	debug("Identity::createKeys: sig prv bytes: " + _object->_sig_prv_bytes.toHex());
 
+	// CRYPTO: create encryption public keys
+	_object->_pub           = _object->_prv->public_key();
+	_object->_pub_bytes     = _object->_pub->public_bytes();
+	debug("Identity::createKeys: pub bytes:     " + _object->_pub_bytes.toHex());
+
 	// CRYPTO: create signature public keys
 	_object->_sig_pub       = _object->_sig_prv->public_key();
 	_object->_sig_pub_bytes = _object->_sig_pub->public_bytes();
@@ -127,20 +127,58 @@ void Identity::load_public_key(const Bytes& pub_bytes) {
 }
 
 bool Identity::load(const char* path) {
-/*
-	try:
-		with open(path, "rb") as key_file:
-			prv_bytes = key_file.read()
-			return self.load_private_key(prv_bytes)
-		return False
-	except Exception as e:
-		RNS.log("Error while loading identity from "+str(path), RNS.LOG_ERROR)
-		RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
-*/
-	// MOCK
-	return true;
+	extreme("Reading identity key from storage...");
+	try {
+		Bytes prv_bytes = OS::read_file(path);
+		if (prv_bytes) {
+			return load_private_key(prv_bytes);
+		}
+		else {
+			return false;
+		}
+	}
+	catch (std::exception& e) {
+		error("Error while loading identity from " + std::string(path));
+		error("The contained exception was: " + std::string(e.what()));
+	}
+	return false;
 }
 
+/*
+Saves the identity to a file. This will write the private key to disk,
+and anyone with access to this file will be able to decrypt all
+communication for the identity. Be very careful with this method.
+
+:param path: The full path specifying where to save the identity.
+:returns: True if the file was saved, otherwise False.
+*/
+bool Identity::to_file(const char* path) {
+	extreme("Writing identity key to storage...");
+	try {
+		return OS::write_file(get_private_key(), path);
+	}
+	catch (std::exception& e) {
+		error("Error while saving identity to " + std::string(path));
+		error("The contained exception was: " + std::string(e.what()));
+	}
+	return false;
+}
+
+
+/*
+Create a new :ref:`RNS.Identity<api-identity>` instance from a file.
+Can be used to load previously created and saved identities into Reticulum.
+
+:param path: The full path to the saved :ref:`RNS.Identity<api-identity>` data
+:returns: A :ref:`RNS.Identity<api-identity>` instance, or *None* if the loaded data was invalid.
+*/
+/*static*/ const Identity Identity::from_file(const char* path) {
+	Identity identity(false);
+	if (identity.load(path)) {
+		return identity;
+	}
+	return {Type::NONE};
+}
 
 /*static*/ void Identity::remember(const Bytes& packet_hash, const Bytes& destination_hash, const Bytes& public_key, const Bytes& app_data /*= {Bytes::NONE}*/) {
 	if (public_key.size() != Type::Identity::KEYSIZE/8) {

src/Identity.h

@@ -90,6 +90,7 @@ namespace RNS {
 			_object->_hexhash = _object->_hash.toHex();
 		};
 		bool load(const char* path);
+		bool to_file(const char* path);
 
 		inline const Bytes get_salt() const { assert(_object); return _object->_hash; }
 		inline const Bytes get_context() const { return {Bytes::NONE}; }
@@ -103,6 +104,7 @@ namespace RNS {
 		void prove(const Packet& packet, const Destination& destination) const;
 		void prove(const Packet& packet) const;
 
+		static const Identity from_file(const char* path);
 		static void remember(const Bytes& packet_hash, const Bytes& destination_hash, const Bytes& public_key, const Bytes& app_data = {Bytes::NONE});
 		static Identity recall(const Bytes& destination_hash);
 		static Bytes recall_app_data(const Bytes& destination_hash);

src/Interface.cpp

@@ -73,3 +73,15 @@ void Interface::process_announce_queue() {
 			RNS.log("The announce queue for this interface has been cleared.", RNS.LOG_ERROR)
 */
 }
+
+void ArduinoJson::convertFromJson(JsonVariantConst src, RNS::Interface& dst) {
+	if (!src.isNull()) {
+		RNS::Bytes hash;
+		hash.assignHex(src.as<const char*>());
+		// Query transport for matching interface
+		dst = Transport::find_interface_from_hash(hash);
+	}
+	else {
+		dst = {RNS::Type::NONE};
+	}
+}

src/Interface.h

@@ -176,3 +176,16 @@ namespace RNS {
 	};
 
 }
+
+namespace ArduinoJson {
+	inline bool convertToJson(const RNS::Interface& src, JsonVariant dst) {
+		if (!src) {
+			return dst.set(nullptr);
+		}
+		return dst.set(src.get_hash().toHex());
+	}
+	void convertFromJson(JsonVariantConst src, RNS::Interface& dst);
+	inline bool canConvertFromJson(JsonVariantConst src, const RNS::Interface&) {
+		return src.is<const char*>() && strlen(src.as<const char*>()) == 64;
+	}
+}

src/Packet.cpp

@@ -554,3 +554,15 @@ void PacketReceipt::check_timeout() {
 		}
 	}
 }
+
+void ArduinoJson::convertFromJson(JsonVariantConst src, RNS::Packet& dst) {
+	if (!src.isNull()) {
+		RNS::Bytes hash;
+		hash.assignHex(src.as<const char*>());
+		// Query transport for matching interface
+		dst = Transport::get_cached_packet(hash);
+	}
+	else {
+		dst = {RNS::Type::NONE};
+	}
+}

src/Packet.h

@@ -274,3 +274,16 @@ namespace RNS {
 	};
 
 }
+
+namespace ArduinoJson {
+	inline bool convertToJson(const RNS::Packet& src, JsonVariant dst) {
+		if (!src) {
+			return dst.set(nullptr);
+		}
+		return dst.set(src.get_hash().toHex());
+	}
+	void convertFromJson(JsonVariantConst src, RNS::Packet& dst);
+	inline bool canConvertFromJson(JsonVariantConst src, const RNS::Packet&) {
+		return src.is<const char*>() && strlen(src.as<const char*>()) == 64;
+	}
+}

src/Reticulum.cpp

@@ -2,6 +2,7 @@
 
 #include "Transport.h"
 #include "Log.h"
+#include "Utilities/OS.h"
 
 #include <RNG.h>
 
@@ -13,6 +14,8 @@
 using namespace RNS;
 using namespace RNS::Type::Reticulum;
 
+/*static*/ std::string Reticulum::storagepath;
+
 /*static*/ bool Reticulum::__transport_enabled = false;
 /*static*/ bool Reticulum::__use_implicit_proof = true;
 /*static*/ bool Reticulum::__allow_probes = false;
@@ -40,6 +43,8 @@ Reticulum::Reticulum() : _object(new Object()) {
 	// CBA TEST Transport
 	__transport_enabled = true;
 
+	Utilities::OS::setup();
+
 #ifdef ARDUINO
 	// Stir in the Ethernet MAC address.
 	//byte mac[6];
@@ -51,9 +56,9 @@ Reticulum::Reticulum() : _object(new Object()) {
 	//RNG.addNoiseSource(noise);
  #endif
 
-/*
-	RNS.vendor.platformutils.platform_checks()
+	//z RNS.vendor.platformutils.platform_checks()
 
+/* TODO
 	if configdir != None:
 		Reticulum.configdir = configdir
 	else:
@@ -73,7 +78,15 @@ Reticulum::Reticulum() : _object(new Object()) {
 	Reticulum.cachepath     = Reticulum.configdir+"/storage/cache"
 	Reticulum.resourcepath  = Reticulum.configdir+"/storage/resources"
 	Reticulum.identitypath  = Reticulum.configdir+"/storage/identities"
+*/
+// CBA MOCK
+#ifdef ARDUINO
+	storagepath = "";
+#else
+	storagepath = ".";
+#endif
 
+/* TODO
 	Reticulum.__transport_enabled = False
 	Reticulum.__use_implicit_proof = True
 	Reticulum.__allow_probes = False

src/Reticulum.h

@@ -4,6 +4,7 @@
 #include "Type.h"
 
 #include <vector>
+#include <string>
 #include <memory>
 #include <stdint.h>
 
@@ -12,6 +13,19 @@ namespace RNS {
 	class Reticulum {
 
 	public:
+
+		//z router           = None
+		//z config           = None
+
+		// The default configuration path will be expanded to a directory
+		// named ".reticulum" inside the current users home directory
+		//z userdir          = os.path.expanduser("~")
+		//z configdir        = None
+		//z configpath       = ""
+		//p storagepath      = ""
+		static std::string storagepath;
+		//z cachepath        = ""
+
 		static bool __transport_enabled;
 		static bool __use_implicit_proof;
 		static bool __allow_probes;

src/Test/Test.cpp

@@ -1,7 +1,5 @@
 #include "Test.h"
 
-#include "Log.h"
-
 void test() {
 
 	testOS();
@@ -9,5 +7,6 @@ void test() {
 	testCollections();
 	testReference();
 	testCrypto();
+	testPersistence();
 
 }

src/Test/Test.h

@@ -6,3 +6,4 @@ void testBytes();
 void testCollections();
 void testReference();
 void testCrypto();
+void testPersistence();

src/Test/TestPersistence.cpp

@@ -0,0 +1,502 @@
+//#include <unity.h>
+
+#include "Transport.h"
+#include "Interfaces/LoRaInterface.h"
+#include "Utilities/Persistence.h"
+#include "Utilities/OS.h"
+#include "Log.h"
+#include "Bytes.h"
+
+#include <ArduinoJson.h>
+
+#include <map>
+#include <vector>
+#include <string>
+#include <assert.h>
+#include <stdio.h>
+
+
+class Test {
+public:
+    Test() {}
+    Test(const char* foo, const char* fee) : _foo(foo), _fee(fee) {}
+	const std::string toString() { return std::string("Test(") + _foo + "," + _fee + ")"; }
+    std::string _foo;
+    std::string _fee;
+};
+
+namespace ArduinoJson {
+	template <>
+	struct Converter<Test> {
+		static bool toJson(const Test& src, JsonVariant dst) {
+			dst["foo"] = src._foo;
+			dst["fee"] = src._fee;
+			return true;
+		}
+
+		static Test fromJson(JsonVariantConst src) {
+			return Test(src["foo"], src["fee"]);
+		}
+
+		static bool checkJson(JsonVariantConst src) {
+			return src["foo"].is<std::string>() && src["fee"].is<std::string>();
+		}
+	};
+}
+
+
+#ifdef ARDUINO
+const char test_file_path[] = "/test_file";
+#else
+const char test_file_path[] = "test_file";
+#endif
+const char test_file_data[] = "test data";
+
+void testWrite() {
+	RNS::Bytes data(test_file_data);
+	if (RNS::Utilities::OS::write_file(data, test_file_path)) {
+		RNS::extreme("wrote: " + std::to_string(data.size()) + " bytes");
+	}
+	else {
+		RNS::extreme("write failed");
+		assert(false);
+	}
+}
+
+void testRead() {
+	RNS::Bytes data = RNS::Utilities::OS::read_file(test_file_path);
+	if (data) {
+		RNS::extreme("read: " + std::to_string(data.size()) + " bytes");
+		RNS::extreme("data: " + data.toString());
+		assert(data.size() == strlen(test_file_data));
+		assert(memcmp(data.data(), test_file_data, strlen(test_file_data)) == 0);
+		//assert(RNS::Utilities::OS::remove_file(test_file_path));
+	}
+	else {
+		RNS::extreme("read failed");
+		assert(false);
+	}
+}
+
+
+#ifdef ARDUINO
+const char test_object_path[] = "/test_object";
+#else
+const char test_object_path[] = "test_object";
+#endif
+
+void testSerializeObject() {
+
+	Test test("bar", "fum");
+
+	StaticJsonDocument<1024> doc;
+	doc["foo"] = test._foo;
+	doc["fee"] = test._fee;
+
+	RNS::Bytes data;
+	size_t size = 1024;
+	//size_t length = serializeJson(doc, data.writable(size), size);
+	size_t length = serializeMsgPack(doc, data.writable(size), size);
+	if (length < size) {
+		data.resize(length);
+	}
+	RNS::extreme("serialized " + std::to_string(length) + " bytes");
+	if (length > 0) {
+		if (RNS::Utilities::OS::write_file(data, test_object_path)) {
+			RNS::extreme("wrote: " + std::to_string(data.size()) + " bytes");
+		}
+		else {
+			RNS::extreme("write failed");
+			assert(false);
+		}
+	}
+	else {
+		RNS::extreme("failed to serialize");
+		assert(false);
+	}
+
+}
+
+void testDeserializeObject() {
+
+	//StaticJsonDocument<8192> doc;
+	DynamicJsonDocument doc(8192);
+
+	RNS::Bytes data = RNS::Utilities::OS::read_file(test_object_path);
+	if (data) {
+		RNS::extreme("read: " + std::to_string(data.size()) + " bytes");
+		//RNS::extreme("data: " + data.toString());
+		//DeserializationError error = deserializeJson(doc, data.data());
+		DeserializationError error = deserializeMsgPack(doc, data.data());
+		if (!error) {
+			JsonObject root = doc.as<JsonObject>();
+			for (JsonPair kv : root) {
+				RNS::extreme("key: " + std::string(kv.key().c_str()) + " value: " + kv.value().as<const char*>());
+			}
+		}
+		else {
+			RNS::extreme("failed to deserialize");
+			assert(false);
+		}
+		//assert(RNS::Utilities::OS::remove_file(test_object_path));
+	}
+	else {
+		RNS::extreme("read failed");
+		assert(false);
+	}
+
+}
+
+
+#ifdef ARDUINO
+const char test_vector_path[] = "/test_vector";
+#else
+const char test_vector_path[] = "test_vector";
+#endif
+
+void testSerializeVector() {
+
+	//std::vector<std::string> vector;
+	//vector.push_back("foo");
+	//vector.push_back("bar");
+	std::vector<Test> vector;
+	vector.push_back({"bar1", "fum1"});
+	vector.push_back({"bar2", "fum2"});
+
+	StaticJsonDocument<4096> doc;
+	//copyArray(vector, doc.createNestedArray("vector"));
+	//doc["vector"] = vector;
+	//copyArray(vector, doc);
+	doc = vector;
+
+/*
+	DynamicJsonBuffer jsonBuffer;
+	JsonObject& root = jsonBuffer.createObject();
+
+	//JsonObject& weather = root.createNestedObject("weather");
+	//weather["temperature"] = 12;
+	//weather["condition"] = "cloudy";
+
+	//JsonArray& coords = root.createNestedArray("coords");
+	//coords.add(48.7507371, 7);
+	//coords.add(2.2625587, 7);
+	JsonArray& arr = root.createNestedArray("vec");
+	for (auto entry : vec) {
+		arr.add(
+	}
+*/
+
+	RNS::Bytes data;
+	size_t size = 4096;
+	size_t length = serializeJson(doc, data.writable(size), size);
+	//size_t length = serializeMsgPack(doc, data.writable(size), size);
+	if (length < size) {
+		data.resize(length);
+	}
+	RNS::extreme("testSerializeVector: serialized " + std::to_string(length) + " bytes");
+	if (length > 0) {
+		if (RNS::Utilities::OS::write_file(data, test_vector_path)) {
+			RNS::extreme("testSerializeVector: wrote: " + std::to_string(data.size()) + " bytes");
+		}
+		else {
+			RNS::extreme("testSerializeVector: write failed");
+			assert(false);
+		}
+	}
+	else {
+		RNS::extreme("testSerializeVector: failed to serialize");
+		assert(false);
+	}
+
+}
+
+void testDeserializeVector() {
+
+	// Compute the required size
+	const int capacity =
+		JSON_ARRAY_SIZE(2) +
+		2*JSON_OBJECT_SIZE(3) +
+		4*JSON_OBJECT_SIZE(1);
+
+	//StaticJsonDocument<8192> doc;
+	DynamicJsonDocument doc(8192);
+
+	RNS::Bytes data = RNS::Utilities::OS::read_file(test_vector_path);
+	if (data) {
+		RNS::extreme("testDeserializeVector: read: " + std::to_string(data.size()) + " bytes");
+		//RNS::extreme("testDeserializeVector: data: " + data.toString());
+		DeserializationError error = deserializeJson(doc, data.data());
+		//DeserializationError error = deserializeMsgPack(doc, data.data());
+		if (!error) {
+			RNS::extreme("testDeserializeVector: successfully deserialized");
+
+			std::vector<Test> vector = doc.as<std::vector<Test>>();
+			for (auto& test : vector) {
+				RNS::extreme("testDeserializeVector: entry: " + test.toString());
+			}
+
+			//JsonArray arr = doc.as<JsonArray>();
+			//for (JsonVariant elem : arr) {
+			//	RNS::extreme("testDeserializeVector: entry: " + elem.as<Test>().toString());
+			//}
+		}
+		else {
+			RNS::extreme("testDeserializeVector: failed to deserialize");
+			assert(false);
+		}
+		//assert(RNS::Utilities::OS::remove_file(test_vector_path));
+	}
+	else {
+		RNS::extreme("testDeserializeVector: read failed");
+		assert(false);
+	}
+
+}
+
+
+#ifdef ARDUINO
+const char test_map_path[] = "/test_map";
+#else
+const char test_map_path[] = "test_map";
+#endif
+
+void testSerializeMap() {
+
+	//std::map<std::string, std::string> map;
+	//map.insert({"foo", "bar"});
+	//map.insert({"fee", "fum"});
+	std::map<std::string, Test> map;
+	map.insert({"one", {"bar1", "fum1"}});
+	map.insert({"two", {"bar2", "fum2"}});
+
+	StaticJsonDocument<4096> doc;
+	//copyArray(map, doc.createNestedArray("map"));
+	//doc["map"] = map;
+	//copyArray(map, doc);
+	doc = map;
+
+	RNS::Bytes data;
+	size_t size = 4096;
+	size_t length = serializeJson(doc, data.writable(size), size);
+	//size_t length = serializeMsgPack(doc, data.writable(size), size);
+	if (length < size) {
+		data.resize(length);
+	}
+	RNS::extreme("testSerializeMap: serialized " + std::to_string(length) + " bytes");
+	if (length > 0) {
+		if (RNS::Utilities::OS::write_file(data, test_map_path)) {
+			RNS::extreme("testSerializeMap: wrote: " + std::to_string(data.size()) + " bytes");
+		}
+		else {
+			RNS::extreme("testSerializeMap: write failed");
+			assert(false);
+		}
+	}
+	else {
+		RNS::extreme("testSerializeMap: failed to serialize");
+		assert(false);
+	}
+
+}
+
+void testDeserializeMap() {
+
+	// Compute the required size
+	const int capacity =
+		JSON_ARRAY_SIZE(2) +
+		2*JSON_OBJECT_SIZE(3) +
+		4*JSON_OBJECT_SIZE(1);
+
+	//StaticJsonDocument<8192> doc;
+	DynamicJsonDocument doc(8192);
+
+	RNS::Bytes data = RNS::Utilities::OS::read_file(test_map_path);
+	if (data) {
+		RNS::extreme("testDeserializeMap: read: " + std::to_string(data.size()) + " bytes");
+		//RNS::extreme("testDeserializeMap: data: " + data.toString());
+		DeserializationError error = deserializeJson(doc, data.data());
+		//DeserializationError error = deserializeMsgPack(doc, data.data());
+		if (!error) {
+			RNS::extreme("testDeserializeMap: successfully deserialized");
+
+			std::map<std::string, Test> map(doc.as<std::map<std::string, Test>>());
+			for (auto& [str, test] : map) {
+				RNS::extreme("testDeserializeMap: entry: " + str + " = " + test.toString());
+			}
+
+			//JsonObject root = doc.as<JsonObject>();
+			//for (JsonPair kv : root) {
+			//	RNS::extreme("testDeserializeMap: entry: " + std::string(kv.key().c_str()) + " = " + kv.value().as<Test>().toString());
+			//}
+		}
+		else {
+			RNS::extreme("testDeserializeMap: failed to deserialize");
+			assert(false);
+		}
+		//assert(RNS::Utilities::OS::remove_file(test_map_path));
+	}
+	else {
+		RNS::extreme("testDeserializeMap: testDeserializeVector: read failed");
+		assert(false);
+	}
+
+}
+
+#ifdef ARDUINO
+const char test_destination_table_path[] = "/test_destination_table";
+#else
+const char test_destination_table_path[] = "test_destination_table";
+#endif
+
+void testSerializeDestinationTable() {
+
+	//RNS::Bytes empty;
+	//std::set<RNS::Bytes> blobs;
+	//RNS::Interface interface({RNS::Type::NONE});
+	RNS::Interfaces::LoRaInterface lora_interface; 
+	//RNS::Packet packet({RNS::Type::NONE});
+	static std::map<RNS::Bytes, RNS::Transport::DestinationEntry> map;
+	//DestinationEntry(double time, const Bytes& received_from, uint8_t announce_hops, double expires, const std::set<Bytes>& random_blobs, Interface& receiving_interface, const Packet& packet) :
+	//RNS::Transport::DestinationEntry entry_one(1.0, empty, 1, 0.0, blobs, interface, packet);
+	RNS::Bytes received;
+	received.assignHex("deadbeef");
+	RNS::Transport::DestinationEntry entry_one;
+	entry_one._timestamp = 1.0;
+	entry_one._received_from = received;
+	entry_one._receiving_interface = lora_interface;
+	RNS::Bytes one;
+	one.assignHex("1111111111111111");
+	map.insert({one, entry_one});
+	//RNS::Transport::DestinationEntry entry_two(2.0, empty, 1, 0.0, blobs, interface, packet);
+	RNS::Transport::DestinationEntry entry_two;
+	entry_two._timestamp = 2.0;
+	entry_two._received_from = received;
+	entry_two._receiving_interface = lora_interface;
+	RNS::Bytes two;
+	two.assignHex("2222222222222222");
+	map.insert({two, entry_two});
+
+	for (auto& [hash, test] : map) {
+		RNS::extreme("testSerializeDestinationTable: entry: " + hash.toHex() + " = (" + std::to_string(test._timestamp) + "," + test._received_from.toHex() + ")");
+	}
+
+	StaticJsonDocument<4096> doc;
+	doc = map;
+
+	RNS::Bytes data;
+	size_t size = 4096;
+	size_t length = serializeJson(doc, data.writable(size), size);
+	//size_t length = serializeMsgPack(doc, data.writable(size), size);
+	if (length < size) {
+		data.resize(length);
+	}
+	RNS::extreme("testSerializeDestinationTable: serialized " + std::to_string(length) + " bytes");
+	if (length > 0) {
+		if (RNS::Utilities::OS::write_file(data, test_destination_table_path)) {
+			RNS::extreme("testSerializeDestinationTable: wrote: " + std::to_string(data.size()) + " bytes");
+		}
+		else {
+			RNS::extreme("testSerializeDestinationTable: write failed");
+			assert(false);
+		}
+	}
+	else {
+		RNS::extreme("testSerializeDestinationTable: failed to serialize");
+		assert(false);
+	}
+
+}
+
+void testDeserializeDestinationTable() {
+
+	// Compute the required size
+	const int capacity =
+		JSON_ARRAY_SIZE(2) +
+		2*JSON_OBJECT_SIZE(3) +
+		4*JSON_OBJECT_SIZE(1);
+
+	//StaticJsonDocument<8192> doc;
+	DynamicJsonDocument doc(8192);
+
+	RNS::Bytes data = RNS::Utilities::OS::read_file(test_destination_table_path);
+	if (data) {
+		RNS::extreme("testDeserializeDestinationTable: read: " + std::to_string(data.size()) + " bytes");
+		//RNS::extreme("testDeserializeVector: data: " + data.toString());
+		DeserializationError error = deserializeJson(doc, data.data());
+		//DeserializationError error = deserializeMsgPack(doc, data.data());
+		if (!error) {
+			RNS::extreme("testDeserializeDestinationTable: successfully deserialized");
+
+			static std::map<RNS::Bytes, RNS::Transport::DestinationEntry> map(doc.as<std::map<RNS::Bytes, RNS::Transport::DestinationEntry>>());
+			for (auto& [hash, test] : map) {
+				RNS::extreme("testDeserializeDestinationTable: entry: " + hash.toHex() + " = (" + std::to_string(test._timestamp) + "," + test._received_from.toHex() + ")");
+			}
+
+			//JsonObject root = doc.as<JsonObject>();
+			//for (JsonPair kv : root) {
+			//	RNS::extreme("testDeserializeDestinationTable: entry: " + std::string(kv.key().c_str()) + " = " + kv.value().as<Test>().toString());
+			//}
+		}
+		else {
+			RNS::extreme("testDeserializeDestinationTable: failed to deserialize");
+			assert(false);
+		}
+		//assert(RNS::Utilities::OS::remove_file(test_destination_table_path));
+	}
+	else {
+		RNS::extreme("testDeserializeDestinationTable: read failed");
+		assert(false);
+	}
+
+}
+
+
+void testPersistence() {
+	RNS::head("Running testPersistence...", RNS::LOG_EXTREME);
+	testWrite();
+	testRead();
+	testSerializeObject();
+	testDeserializeObject();
+	testSerializeVector();
+	testDeserializeVector();
+	testSerializeMap();
+	testDeserializeMap();
+	testSerializeDestinationTable();
+	testDeserializeDestinationTable();
+}
+
+/*
+void setUp(void) {
+	// set stuff up here
+}
+
+void tearDown(void) {
+	// clean stuff up here
+}
+
+int runUnityTests(void) {
+	UNITY_BEGIN();
+	RUN_TEST(testCollections);
+	return UNITY_END();
+}
+
+// For native dev-platform or for some embedded frameworks
+int main(void) {
+	return runUnityTests();
+}
+
+// For Arduino framework
+void setup() {
+	// Wait ~2 seconds before the Unity test runner
+	// establishes connection with a board Serial interface
+	delay(2000);
+
+	runUnityTests();
+}
+void loop() {}
+
+// For ESP-IDF framework
+void app_main() {
+	runUnityTests();
+}
+*/

src/Transport.cpp

@@ -81,15 +81,17 @@ using namespace RNS::Utilities;
 	_owner = reticulum_instance;
 
 	if (!_identity) {
-		//z transport_identity_path = Reticulum::storagepath+"/transport_identity"
-		//z if (os.path.isfile(transport_identity_path)) {
-		//z 	identity = Identity.from_file(transport_identity_path);
-		//z }
+		std::string transport_identity_path = Reticulum::storagepath + "/transport_identity";
+// CBA TEST
+//OS::remove_file(transport_identity_path.c_str());
+		if (OS::file_exists(transport_identity_path.c_str())) {
+			_identity = Identity::from_file(transport_identity_path.c_str());
+		}
 
 		if (!_identity) {
 			verbose("No valid Transport Identity in storage, creating...");
-			_identity = new Identity();
-			//z _identity.to_file(transport_identity_path);
+			_identity = Identity();
+			_identity.to_file(transport_identity_path.c_str());
 		}
 		else {
 			verbose("Loaded Transport Identity from storage");
@@ -135,9 +137,8 @@ using namespace RNS::Utilities;
 // TODO
 /*
 	if RNS.Reticulum.transport_enabled():
-		destination_table_path = RNS.Reticulum.storagepath+"/destination_table"
-		tunnel_table_path = RNS.Reticulum.storagepath+"/tunnels"
 
+		destination_table_path = RNS.Reticulum.storagepath+"/destination_table"
 		if os.path.isfile(destination_table_path) and not Transport.owner.is_connected_to_shared_instance:
 			serialised_destinations = []
 			try:
@@ -183,6 +184,7 @@ using namespace RNS::Utilities;
 			except Exception as e:
 				RNS.log("Could not load destination table from storage, the contained exception was: "+str(e), RNS.LOG_ERROR)
 
+		tunnel_table_path = RNS.Reticulum.storagepath+"/tunnels"
 		if os.path.isfile(tunnel_table_path) and not Transport.owner.is_connected_to_shared_instance:
 			serialised_tunnels = []
 			try:

src/Transport.h

@@ -4,6 +4,8 @@
 #include "Bytes.h"
 #include "Type.h"
 
+#include <ArduinoJson.h>
+
 #include <memory>
 #include <vector>
 #include <list>
@@ -54,11 +56,12 @@ namespace RNS {
     */
 	class Transport {
 
-	private:
+	public:
 		// CBA TODO Analyze safety of using Inrerface references here
 		// CBA TODO Analyze safety of using Packet references here
 		class DestinationEntry {
 		public:
+			DestinationEntry() {}
 			DestinationEntry(double time, const Bytes& received_from, uint8_t announce_hops, double expires, const std::set<Bytes>& random_blobs, Interface& receiving_interface, const Packet& packet) :
 				_timestamp(time),
 				_received_from(received_from),
@@ -74,11 +77,11 @@ namespace RNS {
 			Bytes _received_from;
 			uint8_t _hops = 0;
 			double _expires = 0;
-			std::set<Bytes> _random_blobs;
+			const std::set<Bytes> _random_blobs;
 			// CBA TODO does this need to be a reference in order for virtual method callbacks to work?
-			Interface& _receiving_interface;
+			Interface _receiving_interface = {Type::NONE};
 			//const Packet& _announce_packet;
-			Packet _announce_packet;
+			const Packet _announce_packet = {Type::NONE};
 		};
 
 		// CBA TODO Analyze safety of using Inrerface references here
@@ -101,17 +104,17 @@ namespace RNS {
 			double _timestamp = 0;
 			double _retransmit_timeout = 0;
 			uint8_t _retries = 0;
-			Bytes _received_from;
+			const Bytes _received_from;
 			uint8_t _hops = 0;
 			// CBA Storing packet reference causes memory issues, presumably because orignal packet is being destroyed
 			//  MUST use instance instad of reference!!!
 			//const Packet& _packet;
-			Packet _packet;
+			const Packet _packet = {Type::NONE};
 			uint8_t _local_rebroadcasts = 0;
 			bool _block_rebroadcasts = false;
 			// CBA TODO does this need to be a reference in order for virtual method callbacks to work?
 			//const Interface& _attached_interface;
-			Interface _attached_interface;
+			const Interface _attached_interface = {Type::NONE};
 		};
 
 		// CBA TODO Analyze safety of using Inrerface references here
@@ -131,16 +134,16 @@ namespace RNS {
 			}
 		public:
 			double _timestamp = 0;
-			Bytes _next_hop;
+			const Bytes _next_hop;
 			// CBA TODO does this need to be a reference in order for virtual method callbacks to work?
 			//const Interface& _outbound_interface;
-			Interface _outbound_interface;
+			const Interface _outbound_interface = {Type::NONE};
 			uint8_t _remaining_hops = 0;
 			// CBA TODO does this need to be a reference in order for virtual method callbacks to work?
 			//const Interface& _receiving_interface;
-			Interface _receiving_interface;
+			const Interface _receiving_interface = {Type::NONE};
 			uint8_t _hops = 0;
-			Bytes _destination_hash;
+			const Bytes _destination_hash;
 			bool _validated = false;
 			double _proof_timeout = 0;
 		};
@@ -157,10 +160,10 @@ namespace RNS {
 		public:
 			// CBA TODO does this need to be a reference in order for virtual method callbacks to work?
 			//const Interface& _receiving_interface;
-			Interface _receiving_interface;
+			const Interface _receiving_interface = {Type::NONE};
 			// CBA TODO does this need to be a reference in order for virtual method callbacks to work?
 			//const Interface& _outbound_interface;
-			Interface _outbound_interface;
+			const Interface _outbound_interface = {Type::NONE};
 			double _timestamp = 0;
 		};
 
@@ -174,11 +177,11 @@ namespace RNS {
 			{
 			}
 		public:
-			Bytes _destination_hash;
+			const Bytes _destination_hash;
 			double _timeout = 0;
 			// CBA TODO does this need to be a reference in order for virtual method callbacks to work?
 			//const Interface& _requesting_interface;
-			Interface _requesting_interface;
+			const Interface _requesting_interface = {Type::NONE};
 		};
 
 	public:
@@ -313,3 +316,54 @@ namespace RNS {
 	};
 
 }
+
+namespace ArduinoJson {
+
+	// ArduinoJSON serialization support for RNS::Transport::DestinationEntry
+	template <>
+	struct Converter<RNS::Transport::DestinationEntry> {
+		static bool toJson(const RNS::Transport::DestinationEntry& src, JsonVariant dst) {
+			dst["timestamp"] = src._timestamp;
+			dst["received_from"] = src._received_from;
+			dst["announce_hops"] = src._hops;
+			dst["expires"] = src._expires;
+			//dst["random_blobs"] = src._random_blobs;
+			dst["receiving_interface"] = src._receiving_interface;
+			dst["packet"] = src._announce_packet;
+			return true;
+		}
+		static RNS::Transport::DestinationEntry fromJson(JsonVariantConst src) {
+/**/
+			RNS::Transport::DestinationEntry dst;
+			dst._timestamp = src["timestamp"];
+			dst._received_from = src["received_from"];
+			dst._hops = src["announce_hops"];
+			dst._expires = src["expires"];
+			//dst._random_blobs = src["random_blobs"];
+			dst._receiving_interface = src["receiving_interface"];
+			//dst._announce_packet = src["packet"];
+/**/
+/*
+			//RNS::Transport::DestinationEntry dst(src["timestamp"], src["received_from"], src["announce_hops"], src["expires"], src["random_blobs"], src["receiving_interface"], src["packet"]);
+			RNS::Transport::DestinationEntry dst(
+				src["timestamp"].as<double>(),
+				src["received_from"].as<RNS::Bytes>(),
+				src["announce_hops"].as<int>(),
+				src["expires"].as<double>(),
+				src["random_blobs"].as<std::set<RNS::Bytes>>(),
+				src["receiving_interface"].as<RNS::Interface>(),
+				src["packet"].as<RNS::Packet>()
+			);
+*/
+			return dst;
+		}
+		static bool checkJson(JsonVariantConst src) {
+			return
+				src["timestamp"].is<double>() &&
+				src["received_from"].is<RNS::Bytes>() &&
+				src["announce_hops"].is<int>() &&
+				src["expires"].is<double>();
+		}
+	};
+
+}

src/Utilities/OS.cpp

@@ -0,0 +1,144 @@
+#include "OS.h"
+
+#include "../Log.h"
+
+#ifdef ARDUINO
+#include <FS.h>
+#include <LittleFS.h>
+#else
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#endif
+
+using namespace RNS;
+using namespace RNS::Utilities;
+
+/*static*/ void OS::setup() {
+
+#ifdef ARDUINO
+	// Setup filesystem
+    if (!LittleFS.begin(true)){
+        Serial.println("LittleFS Mount Failed");
+        return;
+    }
+#endif
+
+}
+
+/*static*/ bool OS::file_exists(const char* file_path) {
+#ifdef ARDUINO
+	File file = LittleFS.open(file_path, FILE_READ);
+	if (file) {
+#else
+	FILE* file = fopen(file_path, "r");
+	if (file != nullptr) {
+#endif
+		//RNS::extreme("file_exists: file exists, closing file");
+#ifdef ARDUINO
+		file.close();
+#else
+		fclose(file);
+#endif
+		return true;
+	}
+	else {
+		RNS::extreme("file_exists: failed to open file " + std::string(file_path));
+		return false;
+	}
+}
+
+/*static*/ const Bytes OS::read_file(const char* file_path) {
+    Bytes data;
+#ifdef ARDUINO
+	File file = LittleFS.open(file_path, FILE_READ);
+	if (file) {
+		size_t size = file.size();
+		size_t read = file.readBytes((char*)data.writable(size), size);
+#else
+	FILE* file = fopen(file_path, "r");
+	if (file != nullptr) {
+        fseek(file, 0, SEEK_END);
+        size_t size = ftell(file);
+        rewind(file);
+		//size_t read = fread(data.writable(size), size, 1, file);
+		size_t read = fread(data.writable(size), 1, size, file);
+#endif
+		RNS::extreme("read_file: read " + std::to_string(read) + " bytes from file " + std::string(file_path));
+		if (read != size) {
+			RNS::extreme("read_file: failed to read file " + std::string(file_path));
+            data.clear();
+		}
+		//RNS::extreme("read_file: closing input file");
+#ifdef ARDUINO
+		file.close();
+#else
+		fclose(file);
+#endif
+	}
+	else {
+		RNS::extreme("read_file: failed to open input file " + std::string(file_path));
+	}
+    return data;
+}
+
+/*static*/ bool OS::write_file(const Bytes& data, const char* file_path) {
+    bool success = false;
+#ifdef ARDUINO
+	File file = LittleFS.open(file_path, FILE_WRITE);
+	if (file) {
+		size_t wrote = file.write(data.data(), data.size());
+#else
+	FILE* file = fopen(file_path, "w");
+	if (file != nullptr) {
+        //size_t wrote = fwrite(data.data(), data.size(), 1, file);
+        size_t wrote = fwrite(data.data(), 1, data.size(), file);
+#endif
+        RNS::extreme("write_file: wrote " + std::to_string(wrote) + " bytes to file " + std::string(file_path));
+        if (wrote == data.size()) {
+            success = true;
+        }
+        else {
+			RNS::extreme("write_file: failed to write file " + std::string(file_path));
+		}
+		//RNS::extreme("write_file: closing output file");
+#ifdef ARDUINO
+		file.close();
+#else
+		fclose(file);
+#endif
+	}
+	else {
+		RNS::extreme("write_file: failed to open output file " + std::string(file_path));
+	}
+    return success;
+}
+
+/*static*/ bool OS::remove_file(const char* file_path) {
+#ifdef ARDUINO
+	return LittleFS.remove(file_path);
+#else
+	return (remove(file_path) == 0);
+#endif
+}
+
+/*static*/ bool OS::rename_file(const char* from_file_path, const char* to_file_path) {
+#ifdef ARDUINO
+	return LittleFS.rename(from_file_path, to_file_path);
+#else
+	return (rename(from_file_path, to_file_path) == 0);
+#endif
+}
+
+/*static*/ bool OS::create_directory(const char* directory_path) {
+#ifdef ARDUINO
+	return LittleFS.mkdir(directory_path);
+#else
+	struct stat st = {0};
+	if (stat(directory_path, &st) == 0) {
+		return true;
+	}
+	return (mkdir(directory_path, 0700) == 0);
+#endif
+}

src/Utilities/OS.h

@@ -1,16 +1,24 @@
 #pragma once
 
+#include "../Bytes.h"
+
 #include <cmath>
 #include <unistd.h>
 #include <time.h>
 #include <stdint.h>
 #include <sys/time.h>
 
+#ifdef ARDUINO
+#include "Arduino.h"
+#endif
+
 namespace RNS { namespace Utilities {
 
 	class OS {
 
 	public:
+		static void setup();
+
         // sleep for specified milliseconds
 		//static inline void sleep(float seconds) { ::sleep(seconds); }
 #ifdef ARDUINO
@@ -30,6 +38,13 @@ namespace RNS { namespace Utilities {
         //static inline float round(float value, uint8_t precision) { return std::round(value / precision) * precision; }
         static inline double round(double value, uint8_t precision) { return std::round(value / precision) * precision; }
 
+		static bool file_exists(const char* file_path);
+		static const RNS::Bytes read_file(const char* file_path);
+		static bool write_file(const RNS::Bytes& data, const char* file_path);
+		static bool remove_file(const char* file_path);
+		static bool rename_file(const char* from_file_path, const char* to_file_path);
+		static bool create_directory(const char* directory_path);
+
     };
 
 } }

src/Utilities/Persistence.h

@@ -0,0 +1,89 @@
+
+#include "Bytes.h"
+
+#include <map>
+#include <vector>
+#include <string>
+
+#include <ArduinoJson.h>
+
+namespace ArduinoJson {
+
+	// ArduinoJSON serialization support for std::vector<T>
+	template <typename T>
+		struct Converter<std::vector<T> > {
+		static void toJson(const std::vector<T>& src, JsonVariant dst) {
+			JsonArray array = dst.to<JsonArray>();
+			for (T item : src)
+			array.add(item);
+		}
+		static std::vector<T> fromJson(JsonVariantConst src) {
+			std::vector<T> dst;
+			for (T item : src.as<JsonArrayConst>())
+			dst.push_back(item);
+			return dst;
+		}
+		static bool checkJson(JsonVariantConst src) {
+			JsonArrayConst array = src;
+			bool result = array;
+			for (JsonVariantConst item : array)
+			result &= item.is<T>();
+			return result;
+		}
+	};
+
+	// ArduinoJSON serialization support for std::map<std::string, T>
+	template <typename T>
+	struct Converter<std::map<std::string, T> > {
+		static void toJson(const std::map<std::string, T>& src, JsonVariant dst) {
+			JsonObject obj = dst.to<JsonObject>();
+			for (const auto& item : src)
+			obj[item.first] = item.second;
+		}
+		static std::map<std::string, T> fromJson(JsonVariantConst src) {
+			std::map<std::string, T> dst;
+			for (JsonPairConst item : src.as<JsonObjectConst>())
+			dst[item.key().c_str()] = item.value().as<T>();
+			return dst;
+		}
+		static bool checkJson(JsonVariantConst src) {
+			JsonObjectConst obj = src;
+			bool result = obj;
+			for (JsonPairConst item : obj)
+			result &= item.value().is<T>();
+			return result;
+		}
+	};
+
+	// ArduinoJSON serialization support for std::map<Bytes, T>
+	template <typename T>
+	struct Converter<std::map<RNS::Bytes, T> > {
+		static void toJson(const std::map<RNS::Bytes, T>& src, JsonVariant dst) {
+			JsonObject obj = dst.to<JsonObject>();
+			for (const auto& item : src) {
+			//obj[item.first] = item.second;
+			obj[item.first.toHex()] = item.second;
+			}
+		}
+		static std::map<RNS::Bytes, T> fromJson(JsonVariantConst src) {
+			std::map<RNS::Bytes, T> dst;
+			for (JsonPairConst item : src.as<JsonObjectConst>()) {
+			//dst[item.key().c_str()] = item.value().as<T>();
+			RNS::Bytes key;
+			key.assignHex(item.key().c_str());
+			//dst[key] = item.value().as<T>();
+			dst.insert({key, item.value().as<T>()});
+			}
+			return dst;
+		}
+		static bool checkJson(JsonVariantConst src) {
+			JsonObjectConst obj = src;
+			bool result = obj;
+			for (JsonPairConst item : obj) {
+			result &= item.value().is<T>();
+			}
+			return result;
+		}
+	};
+
+}

src/main.cpp

@@ -120,9 +120,10 @@ void run_tests() {
 		RNS::LogLevel loglevel = RNS::loglevel();
 		//RNS::loglevel(RNS::LOG_WARNING);
 		RNS::loglevel(RNS::LOG_EXTREME);
-		test();
+		//test();
 		//testReference();
 		//testCrypto();
+		testPersistence();
 		RNS::loglevel(loglevel);
 		RNS::extreme("Finished running tests");
 		//return;
@@ -134,6 +135,7 @@ void run_tests() {
 }
 #endif
 
+#if defined(RUN_RETICULUM)
 void reticulum_announce() {
 	if (destination) {
 		RNS::head("Announcing destination...", RNS::LOG_EXTREME);
@@ -145,8 +147,7 @@ void reticulum_announce() {
 	}
 }
 
-#if defined(RUN_RETICULUM)
-void setup_reticulum() {
+void reticulum_setup() {
 	RNS::info("Setting up Reticulum...");
 
 	try {
@@ -177,7 +178,7 @@ void setup_reticulum() {
 
 #ifdef UDP_INTERFACE
 		RNS::head("Starting UDPInterface...", RNS::LOG_EXTREME);
-		udp_interface.start("some_ssid", "some_password", 2424);
+		udp_interface.start("some_ssid", "some_password");
 #endif
 
 #ifdef LORA_INTERFACE
@@ -251,13 +252,14 @@ void setup_reticulum() {
 		destination.receive(recv_packet);
 #endif
 
+		RNS::head("Ready!", RNS::LOG_EXTREME);
 	}
 	catch (std::exception& e) {
-		RNS::error(std::string("!!! Exception in setup_reticulum: ") + e.what() + " !!!");
+		RNS::error(std::string("!!! Exception in reticulum_setup: ") + e.what() + " !!!");
 	}
 }
 
-void teardown_reticulum() {
+void reticulum_teardown() {
 	RNS::info("Tearing down Reticulum...");
 
 	try {
@@ -275,7 +277,7 @@ void teardown_reticulum() {
 
 	}
 	catch (std::exception& e) {
-		RNS::error(std::string("!!! Exception in teardown_reticulum: ") + e.what() + " !!!");
+		RNS::error(std::string("!!! Exception in reticulum_teardown: ") + e.what() + " !!!");
 	}
 }
 #endif
@@ -317,11 +319,11 @@ void setup() {
 #endif
 
 #if defined(RUN_RETICULUM)
-	setup_reticulum();
+	reticulum_setup();
 #endif
 
 #ifdef ARDUINO
-	Serial.print("Goodbye from T-Beam on PlatformIO!\n");
+	//Serial.print("Goodbye from T-Beam on PlatformIO!\n");
 #endif
 }
 
@@ -380,7 +382,9 @@ int main(void) {
 		if (ch > 0) {
 			switch (ch) {
 			case 'a':
+#if defined(RUN_RETICULUM)
 				reticulum_announce();
+#endif
 				break;
 			case 'q':
 				run = false;