Merge branch 'master' of https://github.com/hatkirby/verbly

2 files changed, 167 insertions, 35 deletions

@@ -76,12 +76,24 @@ struct group_t {
   std::list<std::list<framepart_t>> frames;
 };
 
+struct pronunciation_t {
+  std::string phonemes;
+  std::string prerhyme;
+  std::string rhyme;
+  
+  bool operator<(const pronunciation_t& other) const
+  {
+    return phonemes < other.phonemes;
+  }
+};
+
 std::map<std::string, group_t> groups;
 std::map<std::string, verb_t> verbs;
 std::map<std::string, adjective_t> adjectives;
 std::map<std::string, noun_t> nouns;
 std::map<int, std::map<int, int>> wn;
-std::map<std::string, std::set<std::string>> pronunciations;
+std::map<int, int> images;
+std::map<std::string, std::set<pronunciation_t>> pronunciations;
 
 void print_usage()
 {
@@ -89,10 +101,10 @@ void print_usage()
   std::cout << "-------------------------" << std::endl;
   std::cout << "Requires exactly six arguments." << std::endl;
   std::cout << "1. The path to a VerbNet data directory." << std::endl;
-  std::cout << "2. The path to a SemLink vnpbMappings file." << std::endl;
-  std::cout << "3. The path to an AGID infl.txt file." << std::endl;
-  std::cout << "4. The path to a WordNet prolog data directory." << std::endl;
-  std::cout << "5. The path to a CMUDICT pronunciation file." << std::endl;
+  std::cout << "2. The path to an AGID infl.txt file." << std::endl;
+  std::cout << "3. The path to a WordNet prolog data directory." << std::endl;
+  std::cout << "4. The path to a CMUDICT pronunciation file." << std::endl;
+  std::cout << "5. The path to an ImageNet urls.txt file." << std::endl;
   std::cout << "6. Datafile output path." << std::endl;
   
   exit(1);
@@ -431,10 +443,10 @@ int main(int argc, char** argv)
   // Get verbs from AGID
   std::cout << "Reading inflections..." << std::endl;
   
-  std::ifstream agidfile(argv[3]);
+  std::ifstream agidfile(argv[2]);
   if (!agidfile.is_open())
   {
-    std::cout << "Could not open AGID file: " << argv[3] << std::endl;
+    std::cout << "Could not open AGID file: " << argv[2] << std::endl;
     print_usage();
   }
   
@@ -562,10 +574,10 @@ int main(int argc, char** argv)
   // Pronounciations
   std::cout << "Reading pronunciations..." << std::endl;
   
-  std::ifstream pronfile(argv[5]);
+  std::ifstream pronfile(argv[4]);
   if (!pronfile.is_open())
   {
-    std::cout << "Could not open CMUDICT file: " << argv[5] << std::endl;
+    std::cout << "Could not open CMUDICT file: " << argv[4] << std::endl;
     print_usage();
   }
   
@@ -589,10 +601,80 @@ int main(int argc, char** argv)
       std::string canonical(phoneme_data[1]);
       std::transform(std::begin(canonical), std::end(canonical), std::begin(canonical), ::tolower);
       
-      pronunciations[canonical].insert(phoneme_data[2]);
+      std::string phonemes = phoneme_data[2];
+      auto phoneme_set = verbly::split<std::list<std::string>>(phonemes, " ");
+      auto phemstrt = std::find_if(std::begin(phoneme_set), std::end(phoneme_set), [] (std::string phoneme) {
+        return phoneme.find("1") != std::string::npos;
+      });
+      
+      pronunciation_t p;
+      p.phonemes = phonemes;
+      if (phemstrt != std::end(phoneme_set))
+      {
+        std::stringstream rhymer;
+        for (auto it = phemstrt; it != std::end(phoneme_set); it++)
+        {
+          std::string naked;
+          std::remove_copy_if(std::begin(*it), std::end(*it), std::back_inserter(naked), [] (char ch) {
+            return isdigit(ch);
+          });
+          
+          if (it != phemstrt)
+          {
+            rhymer << " ";
+          }
+          
+          rhymer << naked;
+        }
+        
+        p.rhyme = rhymer.str();
+        
+        if (phemstrt != std::begin(phoneme_set))
+        {
+          phemstrt--;
+          p.prerhyme = *phemstrt;
+        } else {
+          p.prerhyme = "";
+        }
+      } else {
+        p.prerhyme = "";
+        p.rhyme = "";
+      }
+      
+      pronunciations[canonical].insert(p);
+    }
+  }
+  
+  // Images
+  std::cout << "Reading images..." << std::endl;
+  
+  std::ifstream imagefile(argv[5]);
+  if (!imagefile.is_open())
+  {
+    std::cout << "Could not open ImageNet file: " << argv[5] << std::endl;
+    print_usage();
+  }
+  
+  for (;;)
+  {
+    std::string line;
+    if (!getline(imagefile, line))
+    {
+      break;
     }
+    
+    if (line.back() == '\r')
+    {
+      line.pop_back();
+    }
+    
+    std::string wnid_s = line.substr(1, 8);
+    int wnid = stoi(wnid_s) + 100000000;
+    images[wnid]++;
   }
   
+  imagefile.close();
+  
   // Start writing output
   std::cout << "Writing schema..." << std::endl;
   
@@ -689,7 +771,7 @@ int main(int argc, char** argv)
       db_error(ppdb, query);
     }
     
-    sqlite3_bind_text(ppstmt, 1, prep.c_str(), prep.length(), SQLITE_STATIC);
+    sqlite3_bind_text(ppstmt, 1, prep.c_str(), prep.length(), SQLITE_TRANSIENT);
     
     if (sqlite3_step(ppstmt) != SQLITE_DONE)
     {
@@ -721,7 +803,7 @@ int main(int argc, char** argv)
       }
       
       sqlite3_bind_int(ppstmt, 1, rowid);
-      sqlite3_bind_text(ppstmt, 2, group.c_str(), group.length(), SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 2, group.c_str(), group.length(), SQLITE_TRANSIENT);
       
       if (sqlite3_step(ppstmt) != SQLITE_DONE)
       {
@@ -744,11 +826,11 @@ int main(int argc, char** argv)
         db_error(ppdb, query);
       }
     
-      sqlite3_bind_text(ppstmt, 1, mapping.second.infinitive.c_str(), mapping.second.infinitive.length(), SQLITE_STATIC);
-      sqlite3_bind_text(ppstmt, 2, mapping.second.past_tense.c_str(), mapping.second.past_tense.length(), SQLITE_STATIC);
-      sqlite3_bind_text(ppstmt, 3, mapping.second.past_participle.c_str(), mapping.second.past_participle.length(), SQLITE_STATIC);
-      sqlite3_bind_text(ppstmt, 4, mapping.second.ing_form.c_str(), mapping.second.ing_form.length(), SQLITE_STATIC);
-      sqlite3_bind_text(ppstmt, 5, mapping.second.s_form.c_str(), mapping.second.s_form.length(), SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 1, mapping.second.infinitive.c_str(), mapping.second.infinitive.length(), SQLITE_TRANSIENT);
+      sqlite3_bind_text(ppstmt, 2, mapping.second.past_tense.c_str(), mapping.second.past_tense.length(), SQLITE_TRANSIENT);
+      sqlite3_bind_text(ppstmt, 3, mapping.second.past_participle.c_str(), mapping.second.past_participle.length(), SQLITE_TRANSIENT);
+      sqlite3_bind_text(ppstmt, 4, mapping.second.ing_form.c_str(), mapping.second.ing_form.length(), SQLITE_TRANSIENT);
+      sqlite3_bind_text(ppstmt, 5, mapping.second.s_form.c_str(), mapping.second.s_form.length(), SQLITE_TRANSIENT);
     
       if (sqlite3_step(ppstmt) != SQLITE_DONE)
       {
@@ -780,14 +862,26 @@ int main(int argc, char** argv)
         
         for (auto pronunciation : pronunciations[canonical])
         {
-          query = "INSERT INTO verb_pronunciations (verb_id, pronunciation) VALUES (?, ?)";
+          if (!pronunciation.rhyme.empty())
+          {
+            query = "INSERT INTO verb_pronunciations (verb_id, pronunciation, prerhyme, rhyme) VALUES (?, ?, ?, ?)";
+          } else {
+            query = "INSERT INTO verb_pronunciations (verb_id, pronunciation) VALUES (?, ?)";
+          }
+          
           if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
           {
             db_error(ppdb, query);
           }
           
           sqlite3_bind_int(ppstmt, 1, rowid);
-          sqlite3_bind_text(ppstmt, 2, pronunciation.c_str(), pronunciation.length(), SQLITE_STATIC);
+          sqlite3_bind_text(ppstmt, 2, pronunciation.phonemes.c_str(), pronunciation.phonemes.length(), SQLITE_TRANSIENT);
+          
+          if (!pronunciation.rhyme.empty())
+          {
+            sqlite3_bind_text(ppstmt, 3, pronunciation.prerhyme.c_str(), pronunciation.prerhyme.length(), SQLITE_TRANSIENT);
+            sqlite3_bind_text(ppstmt, 4, pronunciation.rhyme.c_str(), pronunciation.rhyme.length(), SQLITE_TRANSIENT);
+          }
           
           if (sqlite3_step(ppstmt) != SQLITE_DONE)
           {
@@ -825,7 +919,7 @@ int main(int argc, char** argv)
         db_error(ppdb, query);
       }
       
-      sqlite3_bind_blob(ppstmt, 1, rdm.c_str(), rdm.size(), SQLITE_STATIC);
+      sqlite3_bind_blob(ppstmt, 1, rdm.c_str(), rdm.size(), SQLITE_TRANSIENT);
       
       if (sqlite3_step(ppstmt) != SQLITE_DONE)
       {
@@ -918,7 +1012,7 @@ int main(int argc, char** argv)
         }
         
         sqlite3_bind_int(ppstmt, 1, gid);
-        sqlite3_bind_blob(ppstmt, 2, marshall.c_str(), marshall.length(), SQLITE_STATIC);
+        sqlite3_bind_blob(ppstmt, 2, marshall.c_str(), marshall.length(), SQLITE_TRANSIENT);
         
         if (sqlite3_step(ppstmt) != SQLITE_DONE)
         {
@@ -972,7 +1066,7 @@ int main(int argc, char** argv)
   // - sa: specification (e.g. inaccurate (general) can mean imprecise or incorrect (specific))
   // - sim: synonymy (e.g. cheerful/happy, happy/cheerful)
   // - syntax: positioning flags for some adjectives
-  std::string wnpref {argv[4]};
+  std::string wnpref {argv[3]};
   if (wnpref.back() != '/')
   {
     wnpref += '/';
@@ -1009,7 +1103,7 @@ int main(int argc, char** argv)
     {
       ppgs.update();
       
-      std::regex relation("^s\\(([134]\\d{8}),(\\d+),'([\\w ]+)',");
+      std::regex relation("^s\\(([134]\\d{8}),(\\d+),'(.+)',\\w,\\d+,\\d+\\)\\.$");
       std::smatch relation_data;
       if (!std::regex_search(line, relation_data, relation))
       {
@@ -1019,6 +1113,11 @@ int main(int argc, char** argv)
       int synset_id = stoi(relation_data[1]);
       int wnum = stoi(relation_data[2]);
       std::string word = relation_data[3];
+      size_t word_it;
+      while ((word_it = word.find("''")) != std::string::npos)
+      {
+        word.erase(word_it, 1);
+      }
     
       std::string query;
       switch (synset_id / 100000000)
@@ -1027,9 +1126,9 @@ int main(int argc, char** argv)
         {
           if (nouns.count(word) == 1)
           {
-            query = "INSERT INTO nouns (singular, proper, complexity, plural) VALUES (?, ?, ?, ?)";
+            query = "INSERT INTO nouns (singular, proper, complexity, images, wnid, plural) VALUES (?, ?, ?, ?, ?, ?)";
           } else {
-            query = "INSERT INTO nouns (singular, proper, complexity) VALUES (?, ?, ?)";
+            query = "INSERT INTO nouns (singular, proper, complexity, images, wnid) VALUES (?, ?, ?, ?, ?)";
           }
         
           break;
@@ -1073,7 +1172,7 @@ int main(int argc, char** argv)
         db_error(ppdb, query);
       }
     
-      sqlite3_bind_text(ppstmt, 1, word.c_str(), word.length(), SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 1, word.c_str(), word.length(), SQLITE_TRANSIENT);
       switch (synset_id / 100000000)
       {
         case 1: // Noun
@@ -1083,10 +1182,12 @@ int main(int argc, char** argv)
           }) ? 1 : 0));
           
           sqlite3_bind_int(ppstmt, 3, verbly::split<std::list<std::string>>(word, " ").size());
+          sqlite3_bind_int(ppstmt, 4, images[synset_id]);
+          sqlite3_bind_int(ppstmt, 5, synset_id);
           
           if (nouns.count(word) == 1)
           {
-            sqlite3_bind_text(ppstmt, 4, nouns[word].plural.c_str(), nouns[word].plural.length(), SQLITE_STATIC);
+            sqlite3_bind_text(ppstmt, 6, nouns[word].plural.c_str(), nouns[word].plural.length(), SQLITE_TRANSIENT);
           }
           
           break;
@@ -1099,8 +1200,8 @@ int main(int argc, char** argv)
           
           if (adjectives.count(word) == 1)
           {
-            sqlite3_bind_text(ppstmt, 3, adjectives[word].comparative.c_str(), adjectives[word].comparative.length(), SQLITE_STATIC);
-            sqlite3_bind_text(ppstmt, 4, adjectives[word].superlative.c_str(), adjectives[word].superlative.length(), SQLITE_STATIC);
+            sqlite3_bind_text(ppstmt, 3, adjectives[word].comparative.c_str(), adjectives[word].comparative.length(), SQLITE_TRANSIENT);
+            sqlite3_bind_text(ppstmt, 4, adjectives[word].superlative.c_str(), adjectives[word].superlative.length(), SQLITE_TRANSIENT);
           }
           
           break;
@@ -1140,21 +1241,36 @@ int main(int argc, char** argv)
           {
             case 1: // Noun
             {
-              query = "INSERT INTO noun_pronunciations (noun_id, pronunciation) VALUES (?, ?)";
+              if (!pronunciation.rhyme.empty())
+              {
+                query = "INSERT INTO noun_pronunciations (noun_id, pronunciation, prerhyme, rhyme) VALUES (?, ?, ?, ?)";
+              } else {
+                query = "INSERT INTO noun_pronunciations (noun_id, pronunciation) VALUES (?, ?)";
+              }
               
               break;
             }
             
             case 3: // Adjective
             {
-              query = "INSERT INTO adjective_pronunciations (adjective_id, pronunciation) VALUES (?, ?)";
+              if (!pronunciation.rhyme.empty())
+              {
+                query = "INSERT INTO adjective_pronunciations (adjective_id, pronunciation, prerhyme, rhyme) VALUES (?, ?, ?, ?)";
+              } else {
+                query = "INSERT INTO adjective_pronunciations (adjective_id, pronunciation) VALUES (?, ?)";
+              }
               
               break;
             }
             
             case 4: // Adverb
             {
-              query = "INSERT INTO adverb_pronunciations (adverb_id, pronunciation) VALUES (?, ?)";
+              if (!pronunciation.rhyme.empty())
+              {
+                query = "INSERT INTO adverb_pronunciations (adverb_id, pronunciation, prerhyme, rhyme) VALUES (?, ?, ?, ?)";
+              } else {
+                query = "INSERT INTO adverb_pronunciations (adverb_id, pronunciation) VALUES (?, ?)";
+              }
               
               break;
             }
@@ -1166,7 +1282,13 @@ int main(int argc, char** argv)
           }
         
           sqlite3_bind_int(ppstmt, 1, rowid);
-          sqlite3_bind_text(ppstmt, 2, pronunciation.c_str(), pronunciation.length(), SQLITE_STATIC);
+          sqlite3_bind_text(ppstmt, 2, pronunciation.phonemes.c_str(), pronunciation.phonemes.length(), SQLITE_TRANSIENT);
+          
+          if (!pronunciation.rhyme.empty())
+          {
+            sqlite3_bind_text(ppstmt, 3, pronunciation.prerhyme.c_str(), pronunciation.prerhyme.length(), SQLITE_TRANSIENT);
+            sqlite3_bind_text(ppstmt, 4, pronunciation.rhyme.c_str(), pronunciation.rhyme.length(), SQLITE_TRANSIENT);
+          }
         
           if (sqlite3_step(ppstmt) != SQLITE_DONE)
           {
@@ -2155,7 +2277,7 @@ int main(int argc, char** argv)
         db_error(ppdb, query);
       }
       
-      sqlite3_bind_text(ppstmt, 1, syn.c_str(), 1, SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 1, syn.c_str(), 1, SQLITE_TRANSIENT);
       sqlite3_bind_int(ppstmt, 2, wn[synset_id][wnum]);
       
       if (sqlite3_step(ppstmt) != SQLITE_DONE)
@@ -55,7 +55,9 @@ CREATE TABLE `nouns` (
   `singular` VARCHAR(32) NOT NULL,
   `plural` VARCHAR(32),
   `proper` INTEGER(1) NOT NULL,
-  `complexity` INTEGER NOT NULL
+  `complexity` INTEGER NOT NULL,
+  `images` INTEGER NOT NULL,
+  `wnid` INTEGER NOT NULL
 );
 
 DROP TABLE IF EXISTS `hypernymy`;
@@ -182,6 +184,8 @@ DROP TABLE IF EXISTS `noun_pronunciations`;
 CREATE TABLE `noun_pronunciations` (
   `noun_id` INTEGER NOT NULL,
   `pronunciation` VARCHAR(64) NOT NULL,
+  `prerhyme` VARCHAR(8),
+  `rhyme` VARCHAR(64),
   FOREIGN KEY (`noun_id`) REFERENCES `nouns`(`noun_id`)
 );
 
@@ -189,6 +193,8 @@ DROP TABLE IF EXISTS `verb_pronunciations`;
 CREATE TABLE `verb_pronunciations` (
   `verb_id` INTEGER NOT NULL,
   `pronunciation` VARCHAR(64) NOT NULL,
+  `prerhyme` VARCHAR(8),
+  `rhyme` VARCHAR(64),
   FOREIGN KEY (`verb_id`) REFERENCES `verbs`(`verb_id`)
 );
 
@@ -196,6 +202,8 @@ DROP TABLE IF EXISTS `adjective_pronunciations`;
 CREATE TABLE `adjective_pronunciations` (
   `adjective_id` INTEGER NOT NULL,
   `pronunciation` VARCHAR(64) NOT NULL,
+  `prerhyme` VARCHAR(8),
+  `rhyme` VARCHAR(64),
   FOREIGN KEY (`adjective_id`) REFERENCES `adjectives`(`adjective_id`)
 );
 
@@ -203,6 +211,8 @@ DROP TABLE IF EXISTS `adverb_pronunciations`;
 CREATE TABLE `adverb_pronunciations` (
   `adverb_id` INTEGER NOT NULL,
   `pronunciation` VARCHAR(64) NOT NULL,
+  `prerhyme` VARCHAR(8),
+  `rhyme` VARCHAR(64),
   FOREIGN KEY (`adverb_id`) REFERENCES `adverbs`(`adverb_id`)
 );