Merge pull request #1917

6e679478 mnemonics: sanity checks for word lists (moneromooo-monero)
e98f1114 mnemonics: misc cleanup (moneromooo-monero)

src/mnemonics/dutch.h

@@ -49,9 +49,7 @@ namespace Language
   class Dutch: public Base
   {
   public:
-    Dutch()
+    Dutch(): Base("Dutch", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "aalglad",
         "aalscholver",
         "aambeeld",
@@ -1678,11 +1676,8 @@ namespace Language
         "zwiep",
         "zwijmel",
         "zworen"
-      });
+      }), 4)
-      unique_prefix_length = 4;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Dutch";
       populate_maps();
     }
   };

src/mnemonics/english.h

@@ -49,9 +49,7 @@ namespace Language
   class English: public Base
   {
   public:
-    English()
+    English(): Base("English", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "abbey",
         "abducts",
         "ability",
@@ -1678,11 +1676,8 @@ namespace Language
         "zombie",
         "zones",
         "zoom"
-      });
+      }), 3)
-      unique_prefix_length = 3;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "English";
       populate_maps();
     }
   };

src/mnemonics/french.h

@@ -49,9 +49,7 @@ namespace Language
   class French: public Base
   {
   public:
-    French()
+    French(): Base("French", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "abandon",
         "abattre",
         "aboi",
@@ -1678,11 +1676,8 @@ namespace Language
         "zinc",
         "zone",
         "zoom"
-      });
+      }), 4)
-      unique_prefix_length = 4;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "French";
       populate_maps();
     }
   };

src/mnemonics/german.h

@@ -51,9 +51,7 @@ namespace Language
   class German: public Base
   {
   public:
-    German()
+    German(): Base("German", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "Abakus",
         "Abart",
         "abbilden",
@@ -1680,11 +1678,8 @@ namespace Language
         "Zündung",
         "Zweck",
         "Zyklop"
-      });
+      }), 4)
-      unique_prefix_length = 4;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "German";
       populate_maps();
     }
   };

src/mnemonics/italian.h

@@ -51,9 +51,7 @@ namespace Language
   class Italian: public Base
   {
   public:
-    Italian()
+    Italian(): Base("Italian", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "abbinare",
         "abbonato",
         "abisso",
@@ -1680,11 +1678,8 @@ namespace Language
         "zolfo",
         "zombie",
         "zucchero"
-      });
+      }), 4)
-      unique_prefix_length = 4;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Italian";
       populate_maps();
     }
   };

src/mnemonics/japanese.h

@@ -51,9 +51,7 @@ namespace Language
   class Japanese: public Base
   {
   public:
-    Japanese()
+    Japanese(): Base("Japanese", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "あいこくしん",
         "あいさつ",
         "あいだ",
@@ -1680,11 +1678,8 @@ namespace Language
         "ひさん",
         "びじゅつかん",
         "ひしょ"
-      });
+      }), 3)
-      unique_prefix_length = 3;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Japanese";
       populate_maps();
     }
   };

src/mnemonics/language_base.h

@@ -38,6 +38,7 @@
 #include <vector>
 #include <unordered_map>
 #include <string>
+#include "misc_log_ex.h"
 
 /*!
  * \namespace Language
@@ -73,44 +74,62 @@ namespace Language
   class Base
   {
   protected:
-    std::vector<std::string> *word_list; /*!< A pointer to the array of words */
+    enum {
-    std::unordered_map<std::string, uint32_t> *word_map; /*!< hash table to find word's index */
+      ALLOW_SHORT_WORDS = 1<<0,
-    std::unordered_map<std::string, uint32_t> *trimmed_word_map; /*!< hash table to find word's trimmed index */
+      ALLOW_DUPLICATE_PREFIXES = 1<<1,
+    };
+    const std::vector<std::string> word_list; /*!< A pointer to the array of words */
+    std::unordered_map<std::string, uint32_t> word_map; /*!< hash table to find word's index */
+    std::unordered_map<std::string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
     std::string language_name; /*!< Name of language */
     uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */
     /*!
      * \brief Populates the word maps after the list is ready.
      */
-    void populate_maps()
+    void populate_maps(uint32_t flags = 0)
     {
       int ii;
-      std::vector<std::string>::iterator it;
+      std::vector<std::string>::const_iterator it;
-      for (it = word_list->begin(), ii = 0; it != word_list->end(); it++, ii++)
+      if (word_list.size () != 1626)
+        throw std::runtime_error("Wrong word list length for " + language_name);
+      for (it = word_list.begin(), ii = 0; it != word_list.end(); it++, ii++)
       {
-        (*word_map)[*it] = ii;
+        word_map[*it] = ii;
+        if ((*it).size() < unique_prefix_length)
+        {
+          if (flags & ALLOW_SHORT_WORDS)
+            MWARNING(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
+          else
+            throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
+        }
+        std::string trimmed;
         if (it->length() > unique_prefix_length)
         {
-          (*trimmed_word_map)[utf8prefix(*it, unique_prefix_length)] = ii;
+          trimmed = utf8prefix(*it, unique_prefix_length);
         }
         else
         {
-          (*trimmed_word_map)[*it] = ii;
+          trimmed = *it;
         }
+        if (trimmed_word_map.find(trimmed) != trimmed_word_map.end())
+        {
+          if (flags & ALLOW_DUPLICATE_PREFIXES)
+            MWARNING("Duplicate prefix in " << language_name << " word list: " << trimmed);
+          else
+            throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + trimmed);
+        }
+        trimmed_word_map[trimmed] = ii;
       }
     }
   public:
-    Base()
+    Base(const char *language_name, const std::vector<std::string> &words, uint32_t prefix_length):
+      word_list(words),
+      unique_prefix_length(prefix_length),
+      language_name(language_name)
     {
-      word_list = new std::vector<std::string>;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      unique_prefix_length = 4;
     }
     virtual ~Base()
     {
-      delete word_list;
-      delete word_map;
-      delete trimmed_word_map;
     }
     /*!
      * \brief Returns a pointer to the word list.
@@ -118,7 +137,7 @@ namespace Language
      */
     const std::vector<std::string>& get_word_list() const
     {
-      return *word_list;
+      return word_list;
     }
     /*!
      * \brief Returns a pointer to the word map.
@@ -126,7 +145,7 @@ namespace Language
      */
     const std::unordered_map<std::string, uint32_t>& get_word_map() const
     {
-      return *word_map;
+      return word_map;
     }
     /*!
      * \brief Returns a pointer to the trimmed word map.
@@ -134,13 +153,13 @@ namespace Language
      */
     const std::unordered_map<std::string, uint32_t>& get_trimmed_word_map() const
     {
-      return *trimmed_word_map;
+      return trimmed_word_map;
     }
     /*!
      * \brief Returns the name of the language.
      * \return Name of the language.
      */
-    std::string get_language_name() const
+    const std::string &get_language_name() const
     {
       return language_name;
     }

src/mnemonics/old_english.h

@@ -51,9 +51,7 @@ namespace Language
   class OldEnglish: public Base
   {
   public:
-    OldEnglish()
+    OldEnglish(): Base("OldEnglish", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "like",
         "just",
         "love",
@@ -1680,12 +1678,9 @@ namespace Language
         "unseen",
         "weapon",
         "weary"
-      });
+      }), 4)
-      unique_prefix_length = 4;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
+      populate_maps(ALLOW_DUPLICATE_PREFIXES | ALLOW_SHORT_WORDS);
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "OldEnglish";
-      populate_maps();
     }
   };
 }

src/mnemonics/portuguese.h

@@ -49,9 +49,7 @@ namespace Language
   class Portuguese: public Base
   {
   public:
-    Portuguese()
+    Portuguese(): Base("Portuguese", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "abaular",
         "abdominal",
         "abeto",
@@ -1678,11 +1676,8 @@ namespace Language
         "zeloso",
         "zenite",
         "zumbi"
-      });
+      }), 4)
-      unique_prefix_length = 4;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Portuguese";
       populate_maps();
     }
   };

src/mnemonics/russian.h

@@ -51,9 +51,7 @@ namespace Language
   class Russian: public Base
   {
   public:
-    Russian()
+    Russian(): Base("Russian", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "абажур",
         "абзац",
         "абонент",
@@ -1680,11 +1678,8 @@ namespace Language
         "яхта",
         "ячейка",
         "ящик"
-      });
+      }), 4)
-      unique_prefix_length = 4;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Russian";
       populate_maps();
     }
   };

src/mnemonics/spanish.h

@@ -51,9 +51,7 @@ namespace Language
   class Spanish: public Base
   {
   public:
-    Spanish()
+    Spanish(): Base("Spanish", std::vector<std::string>({
-    {
-      word_list = new std::vector<std::string>({
         "ábaco",
         "abdomen",
         "abeja",
@@ -1680,12 +1678,9 @@ namespace Language
         "risa",
         "ritmo",
         "rito"
-      });
+      }), 4)
-      unique_prefix_length = 4;
+    {
-      word_map = new std::unordered_map<std::string, uint32_t>;
+      populate_maps(ALLOW_SHORT_WORDS);
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Spanish";
-      populate_maps();
     }
   };
 }

tests/unit_tests/mnemonics.cpp

@@ -38,6 +38,11 @@
 #include "mnemonics/spanish.h"
 #include "mnemonics/portuguese.h"
 #include "mnemonics/japanese.h"
+#include "mnemonics/german.h"
+#include "mnemonics/italian.h"
+#include "mnemonics/russian.h"
+#include "mnemonics/french.h"
+#include "mnemonics/dutch.h"
 #include "mnemonics/old_english.h"
 #include "mnemonics/language_base.h"
 #include "mnemonics/singleton.h"
@@ -133,6 +138,19 @@ namespace
   }
 }
 
+TEST(mnemonics, consistency)
+{
+  try {
+    std::vector<std::string> language_list;
+    crypto::ElectrumWords::get_language_list(language_list);
+  }
+  catch(const std::exception &e)
+  {
+    std::cout << "Error initializing mnemonics: " << e.what() << std::endl;
+    ASSERT_TRUE(false);
+  }
+}
+
 TEST(mnemonics, all_languages)
 {
   srand(time(NULL));
@@ -141,12 +159,23 @@ TEST(mnemonics, all_languages)
     Language::Singleton<Language::Spanish>::instance(),
     Language::Singleton<Language::Portuguese>::instance(),
     Language::Singleton<Language::Japanese>::instance(),
+    Language::Singleton<Language::German>::instance(),
+    Language::Singleton<Language::Italian>::instance(),
+    Language::Singleton<Language::Russian>::instance(),
+    Language::Singleton<Language::French>::instance(),
+    Language::Singleton<Language::Dutch>::instance(),
   });
 
   for (std::vector<Language::Base*>::iterator it = languages.begin(); it != languages.end(); it++)
   {
+    try {
       test_language(*(*it));
     }
+    catch (const std::exception &e) {
+      std::cout << "Error testing " << (*it)->get_language_name() << " language: " << e.what() << std::endl;
+      ASSERT_TRUE(false);
+    }
+  }
 }
 
 TEST(mnemonics, language_detection_with_bad_checksum)