fix merge mining with more than one merge mined chain

reported by sech1

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -739,22 +739,28 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
-  bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth)
+  bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, uint64_t mm_merkle_tree_depth)
   {
-    CHECK_AND_ASSERT_MES(mm_merkle_tree_depth < 32, false, "merge mining merkle tree depth should be less than 32");
     size_t start_pos = tx_extra.size();
-    tx_extra.resize(tx_extra.size() + 3 + 32);
+    static const size_t max_varint_size = 16;
+    tx_extra.resize(tx_extra.size() + 2 + 32 + max_varint_size);
     //write tag
     tx_extra[start_pos] = TX_EXTRA_MERGE_MINING_TAG;
     //write data size
     ++start_pos;
-    tx_extra[start_pos] = 33;
-    //write depth varint (always one byte here)
+    const off_t len_bytes = start_pos;
+    // one byte placeholder for length since we'll only know the size later after writing a varint
+    tx_extra[start_pos] = 0;
+    //write depth varint
     ++start_pos;
-    tx_extra[start_pos] = mm_merkle_tree_depth;
+    uint8_t *ptr = &tx_extra[start_pos], *start = ptr;
+    tools::write_varint(ptr, mm_merkle_tree_depth);
     //write data
-    ++start_pos;
+    const size_t varint_size = ptr - start;
+    start_pos += varint_size;
     memcpy(&tx_extra[start_pos], &mm_merkle_root, 32);
+    tx_extra.resize(tx_extra.size() - (max_varint_size - varint_size));
+    tx_extra[len_bytes] = 32 + varint_size;
     return true;
   }
   //---------------------------------------------------------------

src/cryptonote_basic/cryptonote_format_utils.h

@@ -83,7 +83,7 @@ namespace cryptonote
   std::vector<crypto::public_key> get_additional_tx_pub_keys_from_extra(const transaction_prefix& tx);
   bool add_additional_tx_pub_keys_to_extra(std::vector<uint8_t>& tx_extra, const std::vector<crypto::public_key>& additional_pub_keys);
   bool add_extra_nonce_to_tx_extra(std::vector<uint8_t>& tx_extra, const blobdata& extra_nonce);
-  bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth);
+  bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, uint64_t mm_merkle_tree_depth);
   bool remove_field_from_tx_extra(std::vector<uint8_t>& tx_extra, const std::type_info &type);
   void set_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash& payment_id);
   void set_encrypted_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash8& payment_id);

src/cryptonote_basic/merge_mining.cpp

@@ -71,21 +71,21 @@ uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains)
   return path;
 }
 //---------------------------------------------------------------
-uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce)
+uint64_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce)
 {
   CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0");
+  CHECK_AND_ASSERT_THROW_MES(n_aux_chains <= 256, "n_aux_chains is too large");
 
   // how many bits to we need to representing n_aux_chains - 1
   uint32_t n_bits = 1;
-  while ((1u << n_bits) < n_aux_chains && n_bits < 16)
+  while ((1u << n_bits) < n_aux_chains)
     ++n_bits;
-  CHECK_AND_ASSERT_THROW_MES(n_bits <= 16, "Way too many bits required");
 
-  const uint32_t depth = (n_bits - 1) | ((n_aux_chains - 1) << 3) | (nonce << (3 + n_bits));
+  const uint64_t depth = (n_bits - 1) | ((n_aux_chains - 1) << 3) | (((uint64_t)nonce) << (3 + n_bits));
   return depth;
 }
 //---------------------------------------------------------------
-bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce)
+bool decode_mm_depth(uint64_t depth, uint32_t &n_aux_chains, uint32_t &nonce)
 {
   const uint32_t n_bits = 1 + (depth & 7);
   n_aux_chains = 1 + (depth >> 3 & ((1 << n_bits) - 1));

src/cryptonote_basic/merge_mining.h

@@ -36,6 +36,6 @@ namespace cryptonote
 {
   uint32_t get_aux_slot(const crypto::hash &id, uint32_t nonce, uint32_t n_aux_chains);
   uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains);
-  uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce);
-  bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce);
+  uint64_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce);
+  bool decode_mm_depth(uint64_t depth, uint32_t &n_aux_chains, uint32_t &nonce);
 }

src/cryptonote_basic/tx_extra.h

@@ -124,7 +124,7 @@ namespace cryptonote
       END_SERIALIZE()
     };
 
-    size_t depth;
+    uint64_t depth;
     crypto::hash merkle_root;
 
     // load