Merge pull request #8868

a6ce2dc Write interface for new serialization system (Lee *!* Clagett)

contrib/epee/include/serialization/wire.h

@@ -0,0 +1,51 @@
+// Copyright (c) 2021-2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "serialization/wire/fwd.h"
+#include "serialization/wire/read.h"
+#include "serialization/wire/write.h"
+
+//! Define functions that list fields in `type` (using virtual interface)
+#define WIRE_DEFINE_OBJECT(type, map)                          \
+  void read_bytes(::wire::reader& source, type& dest)          \
+  { map(source, dest); }                                       \
+                                                               \
+  void write_bytes(::wire::writer& dest, const type& source)   \
+  { map(dest, source); }
+
+//! Define `from_bytes` and `to_bytes` for `this`.
+#define WIRE_DEFINE_CONVERSIONS()                                       \
+  template<typename R, typename T>                                      \
+  std::error_code from_bytes(T&& source)                                \
+  { return ::wire_read::from_bytes<R>(std::forward<T>(source), *this); } \
+                                                                        \
+  template<typename W, typename T>                                      \
+  std::error_code to_bytes(T& dest) const                               \
+  { return ::wire_write::to_bytes<W>(dest, *this); }
+

contrib/epee/include/serialization/wire/adapted/list.h

@@ -0,0 +1,41 @@
+// Copyright (c) 2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <list>
+#include <type_traits>
+
+#include "serialization/wire/traits.h"
+
+namespace wire
+{
+  template<typename T, typename A>
+  struct is_array<std::list<T, A>>
+    : std::true_type
+  {};
+}

contrib/epee/include/serialization/wire/adapted/vector.h

@@ -0,0 +1,61 @@
+// Copyright (c) 2022-2023-2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+#include <type_traits>
+#include <vector>
+
+#include "byte_slice.h"
+#include "serialization/wire/traits.h"
+
+namespace wire
+{
+  template<typename T, typename A>
+  struct is_array<std::vector<T, A>>
+    : std::true_type
+  {};
+  template<typename A>
+  struct is_array<std::vector<std::uint8_t, A>>
+    : std::false_type
+  {};
+
+  template<typename R, typename A>
+  inline void read_bytes(R& source, std::vector<std::uint8_t, A>& dest)
+  {
+    const epee::byte_slice bytes = source.binary();
+    dest.resize(bytes.size());
+    std::memcpy(dest.data(), bytes.data(), bytes.size());
+  }
+  template<typename W, typename A>
+  inline void write_bytes(W& dest, const std::vector<std::uint8_t, A>& source)
+  {
+    dest.binary(epee::to_span(source));
+  }
+}

contrib/epee/include/serialization/wire/error.h

@@ -0,0 +1,137 @@
+// Copyright (c) 2022-2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <exception>
+#include <system_error>
+#include <type_traits>
+
+#include "misc_log_ex.h"
+
+//! Print default `code` message followed by optional message to debug log then throw `code`.
+#define WIRE_DLOG_THROW_(code, ...)                                     \
+  do                                                                    \
+  {                                                                     \
+    MDEBUG( get_string(code) __VA_ARGS__ );                             \
+    throw ::wire::exception_t<decltype(code)>{code};                    \
+  }                                                                     \
+  while (0)
+
+//! Print default `code` message followed by `msg` to debug log then throw `code`.
+#define WIRE_DLOG_THROW(code, msg)                      \
+  WIRE_DLOG_THROW_(code, << ": " << msg)
+
+namespace wire
+{
+  namespace error
+  {
+    enum class schema : int
+    {
+      none = 0,         //!< Must be zero for `expect<..>`
+      array,            //!< Expected an array value
+      array_max_element,//!< Exceeded max array count
+      array_min_size,   //!< Below min element wire size
+      binary,           //!< Expected a binary value of variable length
+      boolean,          //!< Expected a boolean value
+      enumeration,      //!< Expected a value from a specific set
+      fixed_binary,     //!< Expected a binary value of fixed length
+      integer,          //!< Expected an integer value
+      invalid_key,      //!< Key for object is invalid
+      larger_integer,   //!< Expected a larger integer value
+      maximum_depth,    //!< Hit maximum number of object+array tracking
+      missing_key,      //!< Missing required key for object
+      number,           //!< Expected a number (integer or float) value
+      object,           //!< Expected object value
+      smaller_integer,  //!< Expected a smaller integer value
+      string,           //!< Expected string value
+    };
+
+    //! \return Error message string.
+    const char* get_string(schema value) noexcept;
+
+    //! \return Category for `schema_error`.
+    const std::error_category& schema_category() noexcept;
+
+    //! \return Error code with `value` and `schema_category()`.
+    inline std::error_code make_error_code(const schema value) noexcept
+    {
+      return std::error_code{int(value), schema_category()};
+    }
+  } // error
+
+  //! `std::exception` doesn't require dynamic memory like `std::runtime_error`
+  struct exception : std::exception
+  {
+    exception() noexcept
+      : std::exception()
+    {}
+
+    exception(const exception&) = default;
+    exception& operator=(const exception&) = default;
+    virtual ~exception() noexcept
+    {}
+
+    virtual std::error_code code() const noexcept = 0;
+  };
+
+  template<typename T>
+  class exception_t final : public wire::exception
+  {
+    static_assert(std::is_enum<T>(), "only enumerated types allowed");
+    T value;
+
+  public:
+    exception_t(T value) noexcept
+      : value(value)
+    {}
+
+    exception_t(const exception_t&) = default;
+    ~exception_t() = default;
+    exception_t& operator=(const exception_t&) = default;
+
+    const char* what() const noexcept override final
+    {
+      static_assert(noexcept(noexcept(get_string(value))), "get_string function must be noexcept");
+      return get_string(value);
+    }
+
+    std::error_code code() const noexcept override final
+    {
+      static_assert(noexcept(noexcept(make_error_code(value))), "make_error_code funcion must be noexcept");
+      return make_error_code(value);
+    }
+  };
+} // wire
+
+namespace std
+{
+  template<>
+  struct is_error_code_enum<wire::error::schema>
+    : true_type
+  {};
+}

contrib/epee/include/serialization/wire/field.h

@@ -0,0 +1,141 @@
+// Copyright (c) 2021-2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <functional>
+#include <utility>
+
+#include "serialization/wire/traits.h"
+
+//! A required field has the same key name and C/C++ name
+#define WIRE_FIELD(name)                                \
+  ::wire::field( #name , std::ref( self . name ))
+
+//! A required field has the same key name and C/C++ name AND is cheap to copy (faster output).
+#define WIRE_FIELD_COPY(name)                   \
+  ::wire::field( #name , self . name )
+
+//! The optional field has the same key name and C/C++ name
+#define WIRE_OPTIONAL_FIELD(name)                               \
+  ::wire::optional_field( #name , std::ref( self . name ))
+
+namespace wire
+{
+  /*! Links `name` to a `value` for object serialization.
+
+    `value_type` is `T` with optional `std::reference_wrapper` removed.
+    `value_type` needs a `read_bytes` function when parsing with a
+    `wire::reader` - see `read.h` for more info. `value_type` needs a
+    `write_bytes` function when writing with a `wire::writer` - see `write.h`
+    for more info.
+
+    Any `value_type` where `is_optional_on_empty<value_type> == true`, will
+    automatically be converted to an optional field iff `value_type` has an
+    `empty()` method that returns `true`. The old output engine omitted fields
+    when an array was empty, and the standard input macro would ignore the
+    `false` return for the missing field. For compability reasons, the
+    input/output engine here matches that behavior. See `wrapper/array.h` to
+    enforce a required field even when the array is empty or specialize the
+    `is_optional_on_empty` trait. Only new fields should use this behavior.
+
+    Additional concept requirements for `value_type` when `Required == false`:
+      * must have an `operator*()` function.
+      * must have a conversion to bool function that returns true when
+        `operator*()` is safe to call (and implicitly when the associated field
+        should be written as opposed to skipped/omitted).
+    Additional concept requirements for `value_type` when `Required == false`
+    when reading:
+      * must have an `emplace()` method that ensures `operator*()` is safe to call.
+      * must have a `reset()` method to indicate a field was skipped/omitted.
+
+    If a standard type needs custom serialization, one "trick":
+    ```
+    struct custom_tag{};
+    void read_bytes(wire::reader&, boost::fusion::pair<custom_tag, std::string&>)
+    { ... }
+    void write_bytes(wire::writer&, boost::fusion::pair<custom_tag, const std::string&>)
+    { ... }
+
+    template<typename F, typename T>
+    void object_map(F& format, T& self)
+    {
+      wire::object(format,
+        wire::field("foo", boost::fusion::make_pair<custom_tag>(std::ref(self.foo)))
+      );
+    }
+    ```
+
+    Basically each input/output format needs a unique type so that the compiler
+    knows how to "dispatch" the read/write calls. */
+  template<typename T, bool Required>
+  struct field_
+  {
+    using value_type = unwrap_reference_t<T>;
+
+    //! \return True if field is forced optional when `get_value().empty()`.
+    static constexpr bool optional_on_empty() noexcept
+    { return is_optional_on_empty<value_type>::value; }
+
+    static constexpr bool is_required() noexcept { return Required && !optional_on_empty(); }
+    static constexpr std::size_t count() noexcept { return 1; }
+
+    const char* name;
+    T value;
+
+    constexpr const value_type& get_value() const noexcept { return value; }
+    value_type& get_value() noexcept { return value; }
+  };
+
+  //! Links `name` to `value`. Use `std::ref` if de-serializing.
+  template<typename T>
+  constexpr inline field_<T, true> field(const char* name, T value)
+  {
+    return {name, std::move(value)};
+  }
+
+  //! Links `name` to optional `value`. Use `std::ref` if de-serializing.
+  template<typename T>
+  constexpr inline field_<T, false> optional_field(const char* name, T value)
+  {
+    return {name, std::move(value)};
+  }
+
+
+  template<typename T>
+  inline constexpr bool available(const field_<T, true>& elem)
+  {
+    /* The old output engine always skipped fields when it was an empty array,
+       this follows that behavior. See comments for `field_`. */
+    return elem.is_required() || (elem.optional_on_empty() && !wire::empty(elem.get_value()));
+  }
+  template<typename T>
+  inline constexpr bool available(const field_<T, false>& elem)
+  {
+    return bool(elem.get_value());
+  }
+} // wire

contrib/epee/include/serialization/wire/fwd.h

@@ -0,0 +1,103 @@
+// Copyright (c) 2021-2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <type_traits>
+
+//! Declare an enum to be serialized as an integer
+#define WIRE_AS_INTEGER(type_)                                          \
+  static_assert(std::is_enum<type_>(), "AS_INTEGER only enum types");   \
+  template<typename R>                                                  \
+  inline void read_bytes(R& source, type_& dest)                        \
+  {                                                                     \
+    std::underlying_type<type_>::type temp{};                           \
+    read_bytes(source, temp);                                           \
+    dest = type_(temp);                                                 \
+  }                                                                     \
+  template<typename W>                                                  \
+  inline void write_bytes(W& dest, const type_ source)                  \
+  { write_bytes(dest, std::underlying_type<type_>::type(source)); }
+
+//! Declare functions that list fields in `type` (using virtual interface)
+#define WIRE_DECLARE_OBJECT(type)                       \
+  void read_bytes(::wire::reader&, type&);              \
+  void write_bytes(::wire::writer&, const type&)
+
+//! Cast readers to `rtype` and writers to `wtype` before code expansion
+#define WIRE_BEGIN_MAP_BASE(rtype, wtype)				\
+  template<typename R>                                                  \
+  void read_bytes(R& source)                                            \
+  { wire_map(std::true_type{}, static_cast<rtype&>(source), *this); }	\
+                                                                        \
+  template<typename W>                                                  \
+  void write_bytes(W& dest) const                                       \
+  { wire_map(std::false_type{}, static_cast<wtype&>(dest), *this); }	\
+                                                                        \
+  template<typename B, typename F, typename T>                          \
+  static void wire_map(const B is_read, F& format, T& self)             \
+  { ::wire::object_fwd(is_read, format
+  
+/*! Define `read_bytes`, and `write_bytes` for `this` that forward the
+  derived format types for max performance. */
+#define WIRE_BEGIN_MAP()			\
+  WIRE_BEGIN_MAP_BASE(R, W)
+
+/*! Define `read_bytes`, and `write_bytes` for `this` that forward base format
+  types to reduce code expansion and executable size. */
+#define WIRE_BEGIN_MAP_ASM_SIZE()			\
+  WIRE_BEGIN_MAP_BASE(::wire::reader, ::wire::writer)
+
+//! End object map; omit last `,`
+#define WIRE_END_MAP() );}
+
+namespace wire
+{
+  struct basic_value;
+  class reader;
+  struct writer;
+
+  // defined in `wire/read.h`
+  template<typename R, typename... T>
+  void object_fwd(std::true_type is_read, R& source, T&&... fields);
+
+  // defined in `wire/write.h`
+  template<typename W, typename... T>
+  void object_fwd(std::false_type is_read, W& dest, T... fields);
+}
+namespace wire_read
+{
+  // defined in `wire/read.h`
+  template<typename R, typename T>
+  void bytes(R& source, T&& dest);
+}
+namespace wire_write
+{
+  // defined in `wire/write.h`
+  template<typename W, typename T>
+  void bytes(W& dest, const T& source);
+}

contrib/epee/include/serialization/wire/traits.h

@@ -0,0 +1,195 @@
+// Copyright (c) 2021, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstdint>
+#include <type_traits>
+
+#define WIRE_DECLARE_BLOB_NS(type) \
+  template<>                       \
+  struct is_blob<type>             \
+    : std::true_type               \
+  {}
+
+#define WIRE_DECLARE_BLOB(type)                  \
+  namespace wire { WIRE_DECLARE_BLOB_NS(type); }
+
+#define WIRE_DECLARE_OPTIONAL_ROOT(type) \
+  template<>                             \
+  struct is_optional_root<type>          \
+    : std::true_type                     \
+  {}
+
+namespace wire
+{
+  template<typename T>
+  struct unwrap_reference
+  {
+    using type = std::remove_cv_t<std::remove_reference_t<T>>;
+  };
+
+  template<typename T>
+  struct unwrap_reference<std::reference_wrapper<T>>
+    : std::remove_cv<T>
+  {};
+
+  template<typename T>
+  using unwrap_reference_t = typename unwrap_reference<T>::type;
+
+
+  /*! Mark `T` as an array for writing, and reading when
+    `default_min_element_size<T::value_type>::value != 0`. See `array_` in
+    `wrapper/array.h`. */
+  template<typename T>
+  struct is_array : std::false_type
+  {};
+
+  /*! Mark `T` as fixed binary data for reading+writing. Concept requirements
+      for reading:
+        * `T` must be compatible with `epee::as_mut_byte_span` (`std::is_pod<T>`
+          and no padding).
+      Concept requirements for writing:
+        * `T` must be compatible with `epee::as_byte_span` (std::is_pod<T>` and
+          no padding). */
+  template<typename T>
+  struct is_blob : std::false_type
+  {};
+
+  /*! Forces field to be optional when empty. Concept requirements for `T` when
+    `is_optional_on_empty<T>::value == true`:
+      * must have an `empty()` method that toggles whether the associated
+        `wire::field_<...>` is omitted by the `wire::writer`.
+      * must have a `clear()` method where `empty() == true` upon completion,
+        used by the `wire::reader` when the `wire::field_<...>` is omitted. */
+  template<typename T>
+  struct is_optional_on_empty
+    : is_array<T> // all array types in old output engine were optional when empty
+  {};
+
+  //! When `T` is being read as root object, allow an empty read buffer.
+  template<typename T>
+  struct is_optional_root
+    : std::is_empty<T>
+  {};
+
+  //! A constraint for `wire_read::array` where a max of `N` elements can be read.
+  template<std::size_t N>
+  struct max_element_count
+    : std::integral_constant<std::size_t, N>
+  {
+    // The threshold is low - min_element_size is a better constraint metric
+    static constexpr std::size_t max_bytes() noexcept { return 512 * 1024; } // 512 KiB
+
+    //! \return True if `N` C++ objects of type `T` are below `max_bytes()` threshold.
+    template<typename T>
+    static constexpr bool check() noexcept
+    {
+      return N <= (max_bytes() / sizeof(T));
+    }
+  };
+
+  //! A constraint for `wire_read::array` where each element must use at least `N` bytes on the wire.
+  template<std::size_t N>
+  struct min_element_size
+    : std::integral_constant<std::size_t, N>
+  {
+    static constexpr std::size_t max_ratio() noexcept { return 4; }
+
+    //! \return True if C++ object of type `T` with minimum wire size `N` is below `max_ratio()`.
+    template<typename T>
+    static constexpr bool check() noexcept
+    {
+      return N != 0 ? ((sizeof(T) / N) <= max_ratio()) : false;
+    }
+  };
+
+  /*! Trait used in `wire/read.h` for default `min_element_size` behavior based
+    on an array of `T` objects and `R` reader type. This trait can be used
+    instead of the `wire::array(...)` (and associated macros) functionality, as
+    it sets a global value. The last argument is for `enable_if`. */
+  template<typename R, typename T, typename = void>
+  struct default_min_element_size
+    : std::integral_constant<std::size_t, 0>
+  {};
+
+  //! If `T` is a blob, a safe default for all formats is the size of the blob
+  template<typename R, typename T>
+  struct default_min_element_size<R, T, std::enable_if_t<is_blob<T>::value>>
+    : std::integral_constant<std::size_t, sizeof(T)>
+  {};
+
+  // example usage : `wire::sum(std::size_t(wire::available(fields))...)`
+
+  inline constexpr int sum() noexcept
+  {
+    return 0;
+  }
+  template<typename T, typename... U>
+  inline constexpr T sum(const T head, const U... tail) noexcept
+  {
+    return head + sum(tail...);
+  }
+
+  template<typename... T>
+  using min_element_sizeof = min_element_size<sum(sizeof(T)...)>;
+
+  //! If container has no `reserve(0)` function, this function is used
+  template<typename... T>
+  inline void reserve(const T&...) noexcept
+  {}
+
+  //! Container has `reserve(std::size_t)` function, use it
+  template<typename T>
+  inline auto reserve(T& container, const std::size_t count) -> decltype(container.reserve(count))
+  { return container.reserve(count); }
+
+  //! If `T` has no `empty()` function, this function is used
+  template<typename... T>
+  inline constexpr bool empty(const T&...) noexcept
+  {
+    static_assert(sum(is_optional_on_empty<T>::value...) == 0, "type needs empty method");
+    return false;
+  }
+
+  //! `T` has `empty()` function, use it
+  template<typename T>
+  inline auto empty(const T& container) -> decltype(container.empty())
+  { return container.empty(); }
+
+  //! If `T` has no `clear()` function, this function is used
+  template<typename... T>
+  inline void clear(const T&...) noexcept
+  {
+    static_assert(sum(is_optional_on_empty<T>::value...) == 0, "type needs clear method");
+  }
+
+  //! `T` has `clear()` function, use it
+  template<typename T>
+  inline auto clear(T& container) -> decltype(container.clear())
+  { return container.clear(); }
+} // wire

contrib/epee/include/serialization/wire/wrapper/defaulted.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2021-2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <functional>
+#include <utility>
+
+#include "serialization/wire/field.h"
+#include "serialization/wire/traits.h"
+
+//! An optional field that is omitted when a default value is used
+#define WIRE_FIELD_DEFAULTED(name, default_)                            \
+  ::wire::optional_field( #name , ::wire::defaulted(std::ref( self . name ), default_ ))
+
+namespace wire
+{
+  /*! A wrapper that tells `wire::writer`s to skip field generation when default
+    value, and tells `wire::reader`s to use default value when field not present. */
+  template<typename T, typename U>
+  struct defaulted_
+  {
+    using value_type = unwrap_reference_t<T>;
+
+    T value;
+    U default_;
+
+    constexpr const value_type& get_value() const noexcept { return value; }
+    value_type& get_value() noexcept { return value; }
+
+    // concept requirements for optional fields
+
+    constexpr explicit operator bool() const { return get_value() != default_; }
+    value_type& emplace() noexcept { return get_value(); }
+
+    constexpr const value_type& operator*() const noexcept { return get_value(); }
+    value_type& operator*() noexcept { return get_value(); }
+
+    void reset() { get_value() = default_; }
+  };
+
+  //! Links `value` with `default_`.
+  template<typename T, typename U>
+  inline constexpr defaulted_<T, U> defaulted(T value, U default_)
+  {
+    return {std::move(value), std::move(default_)};
+  }
+
+  /* read/write functions not needed since `defaulted_` meets the concept
+     requirements for an optional type (optional fields are handled
+     directly by the generic read/write code because the field name is omitted
+     entirely when the value is "empty"). */
+} // wire

contrib/epee/include/serialization/wire/write.h

@@ -0,0 +1,287 @@
+// Copyright (c) 2023, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/utility/string_ref.hpp>
+#include <boost/range/size.hpp>
+#include <cstdint>
+#include <system_error>
+#include <type_traits>
+
+#include "byte_slice.h"
+#include "serialization/wire/error.h"
+#include "serialization/wire/field.h"
+#include "serialization/wire/traits.h"
+#include "span.h"
+
+/*
+  Custom types (e.g type `type` in namespace `ns`) can define an output function by:
+    * `namespace wire { template<> struct is_array<ns::type> : std::true_type {}; }`
+    * `namespace wire { template<> struct is_blob<ns::type> : std::true_type {}; }`
+    * `namespace wire { void write_bytes(writer&, const ns::type&); }`
+    * `namespace ns { void write_bytes(wire::writer&, const type&); }`
+
+  See `wrappers.h` for `is_array` requirements, and `traits.h` for `is_blob`
+  requirements. `write_bytes` function can also specify derived type for faster
+  output (i.e.  `namespace ns { void write_bytes(wire::epee_writer&, type&); }`).
+  Using the derived type allows the compiler to de-virtualize and allows for
+  custom functions not defined by base interface. Using base interface allows
+  for multiple formats with minimal instruction count. */
+
+namespace wire
+{
+  //! Interface for converting C/C++ objects to "wire" (byte) formats.
+  struct writer
+  {
+    writer() = default;
+
+    virtual ~writer() noexcept;
+
+    //! By default, insist on retrieving array size before writing array
+    static constexpr std::true_type need_array_size() noexcept { return{}; }
+
+    virtual void boolean(bool) = 0;
+
+    virtual void integer(std::intmax_t) = 0;
+    virtual void unsigned_integer(std::uintmax_t) = 0;
+
+    virtual void real(double) = 0;
+
+    virtual void string(boost::string_ref) = 0;
+    virtual void binary(epee::span<const std::uint8_t>) = 0;
+
+    virtual void start_array(std::size_t) = 0;
+    virtual void end_array() = 0;
+
+    virtual void start_object(std::size_t) = 0;
+    virtual void key(boost::string_ref) = 0;
+    virtual void binary_key(epee::span<const std::uint8_t>) = 0;
+    virtual void end_object() = 0;
+
+  protected:
+    writer(const writer&) = default;
+    writer(writer&&) = default;
+    writer& operator=(const writer&) = default;
+    writer& operator=(writer&&) = default;
+  };
+
+  template<typename W>
+  inline void write_arithmetic(W& dest, const bool source)
+  { dest.boolean(source); }
+
+  template<typename W>
+  inline void write_arithmetic(W& dest, const int source)
+  { dest.integer(source); }
+
+  template<typename W>
+  inline void write_arithmetic(W& dest, const long source)
+  { dest.integer(std::intmax_t(source)); }
+
+  template<typename W>
+  inline void write_arithmetic(W& dest, const long long source)
+  { dest.integer(std::intmax_t(source)); }
+
+  template<typename W>
+  inline void write_arithmetic(W& dest, const unsigned source)
+  { dest.unsigned_integer(source); }
+
+  template<typename W>
+  inline void write_arithmetic(W& dest, const unsigned long source)
+  { dest.unsigned_integer(std::uintmax_t(source)); }
+
+  template<typename W>
+  inline void write_arithmetic(W& dest, const unsigned long long source)
+  { dest.unsigned_integer(std::uintmax_t(source));}
+
+  template<typename W>
+  inline void write_arithmetic(W& dest, const double source)
+  { dest.real(source); }
+
+  // Template both arguments to allow derived writer specializations
+  template<typename W, typename T>
+  inline std::enable_if_t<std::is_arithmetic<T>::value> write_bytes(W& dest, const T source)
+  { write_arithmetic(dest, source); }
+
+  template<typename W>
+  inline void write_bytes(W& dest, const boost::string_ref source)
+  { dest.string(source); }
+
+  template<typename W, typename T>
+  inline std::enable_if_t<is_blob<T>::value> write_bytes(W& dest, const T& source)
+  { dest.binary(epee::as_byte_span(source)); }
+
+  template<typename W>
+  inline void write_bytes(W& dest, const epee::span<const std::uint8_t> source)
+  { dest.binary(source); }
+
+  template<typename W>
+  inline void write_bytes(W& dest, const epee::byte_slice& source)
+  { write_bytes(dest, epee::to_span(source)); }
+
+  //! Use `write_bytes(...)` method if available for `T`.
+  template<typename W, typename T>
+  inline auto write_bytes(W& dest, const T& source) -> decltype(source.write_bytes(dest))
+  { return source.write_bytes(dest); }
+}
+
+namespace wire_write
+{
+  /*! Don't add a function called `write_bytes` to this namespace, it will
+      prevent ADL lookup. ADL lookup delays the function searching until the
+      template is used instead of when its defined. This allows the unqualified
+      calls to `write_bytes` in this namespace to "find" user functions that are
+      declared after these functions. */
+
+  template<typename W, typename T>
+  inline void bytes(W& dest, const T& source)
+  {
+    write_bytes(dest, source); // ADL (searches every associated namespace)
+  }
+
+  //! Use writer `W` to convert `source` into bytes appended to `dest`.
+  template<typename W, typename T, typename U>
+  inline std::error_code to_bytes(T& dest, const U& source)
+  {
+    try
+    {
+      W out{std::move(dest)};
+      bytes(out, source);
+      dest = out.take_buffer();
+    }
+    catch (const wire::exception& e)
+    {
+      dest.clear();
+      return e.code();
+    }
+    catch (...)
+    {
+      dest.clear();
+      throw;
+    }
+    return {};
+  }
+
+  template<typename T>
+  inline std::size_t array_size(std::true_type, const T& source)
+  { return boost::size(source); }
+
+  template<typename T>
+  inline constexpr std::size_t array_size(std::false_type, const T&) noexcept
+  { return 0; }
+
+  template<typename W, typename T>
+  inline void array(W& dest, const T& source)
+  {
+    using value_type = typename T::value_type;
+    static_assert(!std::is_same<value_type, char>::value, "write array of chars as string");
+    static_assert(!std::is_same<value_type, std::int8_t>::value, "write array of signed chars as binary");
+    static_assert(!std::is_same<value_type, std::uint8_t>::value, "write array of unsigned chars as binary");
+
+    dest.start_array(array_size(dest.need_array_size(), source));
+    for (const auto& elem : source)
+      bytes(dest, elem);
+    dest.end_array();
+  }
+
+  template<typename W, typename T>
+  inline bool field(W& dest, const wire::field_<T, true>& field)
+  {
+    // Arrays always optional, see `wire/field.h`
+    if (wire::available(field))
+    {
+      dest.key(field.name);
+      bytes(dest, field.get_value());
+    }
+    return true;
+  }
+
+  template<typename W, typename T>
+  inline bool field(W& dest, const wire::field_<T, false>& field)
+  {
+    if (wire::available(field))
+    {
+      dest.key(field.name);
+      bytes(dest, *field.get_value());
+    }
+    return true;
+  }
+
+  template<typename W, typename T>
+  inline std::enable_if_t<std::is_pod<T>::value> dynamic_object_key(W& dest, const T& source)
+  {
+    dest.binary_key(epee::as_byte_span(source));
+  }
+
+  template<typename W>
+  inline void dynamic_object_key(W& dest, const boost::string_ref source)
+  {
+    dest.key(source);
+  }
+
+  template<typename W, typename T>
+  inline void dynamic_object(W& dest, const T& source)
+  {
+    dest.start_object(source.size());
+    for (const auto& elem : source)
+    {
+      dynamic_object_key(dest, elem.first);
+      bytes(dest, elem.second);
+    }
+    dest.end_object();
+  }
+
+  template<typename W, typename... T>
+  inline void object(W& dest, T&&... fields)
+  {
+    dest.start_object(wire::sum(std::size_t(wire::available(fields))...));
+    const bool dummy[] = {field(dest, std::forward<T>(fields))...};
+    dest.end_object();
+    (void)dummy; // expand into array to get 0,1,2,etc order
+  }
+} // wire_write
+
+namespace wire
+{
+  template<typename W, typename T>
+  inline std::enable_if_t<is_array<T>::value> write_bytes(W& dest, const T& source)
+  {
+    wire_write::array(dest, source);
+  }
+
+  template<typename W, typename... T>
+  inline std::enable_if_t<std::is_base_of<writer, W>::value> object(W& dest, T... fields)
+  {
+    wire_write::object(dest, std::move(fields)...);
+  }
+
+  template<typename W, typename... T>
+  inline void object_fwd(const std::false_type /* is_read */, W& dest, T... fields)
+  {
+    wire::object(dest, std::move(fields)...);
+  }
+}