De-duped pronunciations in generated database hkutil

Identical pronunciations will now share an idea and be re-used by multiple forms. This has a negligible effect on database size, but it's useful for writing queries looking for words with the exact same pronunciations.

This constitutes a minor database update, which we will call d1.2.

2 files changed, 10 insertions, 3 deletions

@@ -573,9 +573,15 @@ namespace verbly {
           }
 
           std::string phonemes = phoneme_data[2];
-          pronunciations_.emplace_back(phonemes);
-          pronunciation& p = pronunciations_.back();
-          formByText_.at(canonical)->addPronunciation(p);
+          if (pronunciationByPhonemes_.count(phonemes)) {
+            pronunciation& p = *pronunciationByPhonemes_[phonemes];
+            formByText_.at(canonical)->addPronunciation(p);
+          } else {
+            pronunciations_.emplace_back(phonemes);
+            pronunciation& p = pronunciations_.back();
+            pronunciationByPhonemes_[phonemes] = &p;
+            formByText_.at(canonical)->addPronunciation(p);
+          }
         }
       }
     }
@@ -139,6 +139,7 @@ namespace verbly {
       std::map<std::string, std::set<word*>> wordsByBaseForm_;
       std::map<std::string, lemma*> lemmaByBaseForm_;
       std::map<std::string, form*> formByText_;
+      std::map<std::string, pronunciation*> pronunciationByPhonemes_;
 
       // Caches