1 files changed, 1186 insertions, 117 deletions

@@ -9,6 +9,8 @@
 #include <sqlite3.h>
 #include <sstream>
 #include <regex>
+#include <list>
+#include "progress.h"
 
 struct verb {
   std::string infinitive;
@@ -18,6 +20,17 @@ struct verb {
   std::string s_form;
 };
 
+struct adjective {
+  std::string base;
+  std::string comparative;
+  std::string superlative;
+};
+
+struct noun {
+  std::string singular;
+  std::string plural;
+};
+
 struct group {
   std::string id;
   std::set<std::string> members;
@@ -25,21 +38,33 @@ struct group {
 
 std::map<std::string, group> groups;
 std::map<std::string, verb> verbs;
+std::map<std::string, adjective> adjectives;
+std::map<std::string, noun> nouns;
 std::map<int, std::map<int, int>> wn;
+std::map<std::string, std::set<std::string>> pronunciations;
 
 void print_usage()
 {
   std::cout << "Verbly Datafile Generator" << std::endl;
   std::cout << "-------------------------" << std::endl;
-  std::cout << "Requires exactly four arguments." << std::endl;
+  std::cout << "Requires exactly six arguments." << std::endl;
   std::cout << "1. The path to a VerbNet data directory." << std::endl;
   std::cout << "2. The path to a SemLink vnpbMappings file." << std::endl;
   std::cout << "3. The path to an AGID infl.txt file." << std::endl;
   std::cout << "4. The path to a WordNet prolog data directory." << std::endl;
-  std::cout << "5. Datafile output path." << std::endl;
+  std::cout << "5. The path to a CMUDICT pronunciation file." << std::endl;
+  std::cout << "6. Datafile output path." << std::endl;
   
   exit(1);
 }
+
+void db_error(sqlite3* ppdb, std::string)
+{
+  std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
+  sqlite3_close_v2(ppdb);
+  print_usage();
+}
+
 /*
 void parse_group(xmlNodePtr top, std::string filename)
 {
@@ -87,7 +112,7 @@ void parse_group(xmlNodePtr top, std::string filename)
 
 int main(int argc, char** argv)
 {
-  if (argc != 6)
+  if (argc != 7)
   {
     print_usage();
   }
@@ -137,7 +162,7 @@ int main(int argc, char** argv)
   closedir(dir);*/
   
   // Get verbs from AGID
-  std::cout << "Reading verb inflection..." << std::endl;
+  std::cout << "Reading inflections..." << std::endl;
   
   std::ifstream agidfile(argv[3]);
   if (!agidfile.is_open())
@@ -162,11 +187,7 @@ int main(int argc, char** argv)
     int divider = line.find_first_of(" ");
     std::string word = line.substr(0, divider);
     line = line.substr(divider+1);
-    
-    if (line[0] != 'V')
-    {
-      continue;
-    }
+    char type = line[0];
     
     if (line[1] == '?')
     {
@@ -174,7 +195,7 @@ int main(int argc, char** argv)
     } else {
       line.erase(0, 3);
     }
-    
+
     std::vector<std::string> forms;
     while (!line.empty())
     {
@@ -187,52 +208,129 @@ int main(int argc, char** argv)
         inflection = line;
         line = "";
       }
-      
+  
       if ((divider = inflection.find_first_of(",?")) != std::string::npos)
       {
         inflection = inflection.substr(0, divider);
       }
-      
+  
       forms.push_back(inflection);
     }
     
-    verb v;
-    v.infinitive = word;
-    if (forms.size() == 4)
+    switch (type)
     {
-      v.past_tense = forms[0];
-      v.past_participle = forms[1];
-      v.ing_form = forms[2];
-      v.s_form = forms[3];
-    } else if (forms.size() == 3)
+      case 'V':
+      {
+        verb v;
+        v.infinitive = word;
+        if (forms.size() == 4)
+        {
+          v.past_tense = forms[0];
+          v.past_participle = forms[1];
+          v.ing_form = forms[2];
+          v.s_form = forms[3];
+        } else if (forms.size() == 3)
+        {
+          v.past_tense = forms[0];
+          v.past_participle = forms[0];
+          v.ing_form = forms[1];
+          v.s_form = forms[2];
+        } else if (forms.size() == 8)
+        {
+          // As of AGID 2014.08.11, this is only "to be"
+          v.past_tense = forms[0];
+          v.past_participle = forms[2];
+          v.ing_form = forms[3];
+          v.s_form = forms[4];
+        } else {
+          // Words that don't fit the cases above as of AGID 2014.08.11:
+          // - may and shall do not conjugate the way we want them to
+          // - methinks only has a past tense and is an outlier
+          // - wit has five forms, and is archaic/obscure enough that we can ignore it for now
+          std::cout << "Ignoring verb \"" << word << "\" due to non-standard number of forms." << std::endl;
+        }
+    
+        verbs[word] = v;
+        
+        break;
+      }
+      
+      case 'A':
+      {
+        adjective adj;
+        adj.base = word;
+        if (forms.size() == 2)
+        {
+          adj.comparative = forms[0];
+          adj.superlative = forms[1];
+        } else {
+          // As of AGID 2014.08.11, this is only "only", which has only the form "onliest"
+          std::cout << "Ignoring adjective/adverb \"" << word << "\" due to non-standard number of forms." << std::endl;
+        }
+        
+        adjectives[word] = adj;
+        
+        break;
+      }
+      
+      case 'N':
+      {
+        noun n;
+        n.singular = word;
+        if (forms.size() == 1)
+        {
+          n.plural = forms[0];
+        } else {
+          // As of AGID 2014.08.11, this is non-existent.
+          std::cout << "Ignoring noun \"" << word << "\" due to non-standard number of forms." << std::endl;
+        }
+        
+        nouns[word] = n;
+        
+        break;
+      }
+    }
+  }
+  
+  // Pronounciations
+  std::cout << "Reading pronunciations..." << std::endl;
+  
+  std::ifstream pronfile(argv[5]);
+  if (!pronfile.is_open())
+  {
+    std::cout << "Could not open CMUDICT file: " << argv[5] << std::endl;
+    print_usage();
+  }
+  
+  for (;;)
+  {
+    std::string line;
+    if (!getline(pronfile, line))
     {
-      v.past_tense = forms[0];
-      v.past_participle = forms[0];
-      v.ing_form = forms[1];
-      v.s_form = forms[2];
-    } else if (forms.size() == 8)
+      break;
+    }
+    
+    if (line.back() == '\r')
     {
-      // As of AGID 2014.08.11, this is only "to be"
-      v.past_tense = forms[0];
-      v.past_participle = forms[2];
-      v.ing_form = forms[3];
-      v.s_form = forms[4];
-    } else {
-      // Words that don't fit the cases above as of AGID 2014.08.11:
-      // - may and shall do not conjugate the way we want them to
-      // - methinks only has a past tense and is an outlier
-      // - wit has five forms, and is archaic/obscure enough that we can ignore it for now
-      std::cout << "Ignoring verb \"" << word << "\" due to non-standard number of forms." << std::endl;
+      line.pop_back();
     }
     
-    verbs[word] = v;
+    std::regex phoneme("([A-Z][^ \\(]*)(?:\\(\\d+\\))?  ([A-Z 0-9]+)");
+    std::smatch phoneme_data;
+    if (std::regex_search(line, phoneme_data, phoneme))
+    {
+      std::string canonical(phoneme_data[1]);
+      std::transform(std::begin(canonical), std::end(canonical), std::begin(canonical), ::tolower);
+      
+      pronunciations[canonical].insert(phoneme_data[2]);
+    }
   }
   
   // Start writing output
-  std::cout << "Writing output..." << std::endl;
+  std::cout << "Writing schema..." << std::endl;
   
   sqlite3* ppdb;
-  if (sqlite3_open_v2(argv[5], &ppdb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL) != SQLITE_OK)
+  if (sqlite3_open_v2(argv[6], &ppdb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL) != SQLITE_OK)
   {
     std::cout << "Error opening output datafile: " << sqlite3_errmsg(ppdb) << std::endl;
     print_usage();
@@ -278,47 +376,82 @@ int main(int argc, char** argv)
     sqlite3_stmt* schmstmt;
     if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &schmstmt, NULL) != SQLITE_OK)
     {
-      std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
-      sqlite3_close_v2(ppdb);
-      print_usage();
+      db_error(ppdb, query);
     }
   
     if (sqlite3_step(schmstmt) != SQLITE_DONE)
     {
-      std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
-      sqlite3_close_v2(ppdb);
-      print_usage();
+      db_error(ppdb, query);
     }
   
     sqlite3_finalize(schmstmt);
   }
   
-  std::cout << "Writing verbs..." << std::endl;
-  for (auto& mapping : verbs)
   {
-    sqlite3_stmt* ppstmt;
-    std::string query("INSERT INTO verbs (infinitive, past_tense, past_participle, ing_form, s_form) VALUES (?, ?, ?, ?, ?)");
-    if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+    progress ppgs("Writing verbs...", verbs.size());
+    for (auto& mapping : verbs)
     {
-      std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
-      sqlite3_close_v2(ppdb);
-      print_usage();
-    }
+      sqlite3_stmt* ppstmt;
+      std::string query("INSERT INTO verbs (infinitive, past_tense, past_participle, ing_form, s_form) VALUES (?, ?, ?, ?, ?)");
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
     
-    sqlite3_bind_text(ppstmt, 1, mapping.second.infinitive.c_str(), mapping.second.infinitive.length(), SQLITE_STATIC);
-    sqlite3_bind_text(ppstmt, 2, mapping.second.past_tense.c_str(), mapping.second.past_tense.length(), SQLITE_STATIC);
-    sqlite3_bind_text(ppstmt, 3, mapping.second.past_participle.c_str(), mapping.second.past_participle.length(), SQLITE_STATIC);
-    sqlite3_bind_text(ppstmt, 4, mapping.second.ing_form.c_str(), mapping.second.ing_form.length(), SQLITE_STATIC);
-    sqlite3_bind_text(ppstmt, 5, mapping.second.s_form.c_str(), mapping.second.s_form.length(), SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 1, mapping.second.infinitive.c_str(), mapping.second.infinitive.length(), SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 2, mapping.second.past_tense.c_str(), mapping.second.past_tense.length(), SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 3, mapping.second.past_participle.c_str(), mapping.second.past_participle.length(), SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 4, mapping.second.ing_form.c_str(), mapping.second.ing_form.length(), SQLITE_STATIC);
+      sqlite3_bind_text(ppstmt, 5, mapping.second.s_form.c_str(), mapping.second.s_form.length(), SQLITE_STATIC);
     
-    if (sqlite3_step(ppstmt) != SQLITE_DONE)
-    {
-      std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
-      sqlite3_close_v2(ppdb);
-      print_usage();
-    }
+      if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      {
+        db_error(ppdb, query);
+      }
     
-    sqlite3_finalize(ppstmt);
+      sqlite3_finalize(ppstmt);
+      
+      std::string canonical(mapping.second.infinitive);
+      std::transform(std::begin(canonical), std::end(canonical), std::begin(canonical), ::tolower);
+      if (pronunciations.count(canonical) == 1)
+      {
+        query = "SELECT last_insert_rowid()";
+        if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+        {
+          db_error(ppdb, query);
+        }
+    
+        if (sqlite3_step(ppstmt) != SQLITE_ROW)
+        {
+          db_error(ppdb, query);
+        }
+    
+        int rowid = sqlite3_column_int(ppstmt, 0);
+    
+        sqlite3_finalize(ppstmt);
+        
+        for (auto pronunciation : pronunciations[canonical])
+        {
+          query = "INSERT INTO verb_pronunciations (verb_id, pronunciation) VALUES (?, ?)";
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, rowid);
+          sqlite3_bind_text(ppstmt, 2, pronunciation.c_str(), pronunciation.length(), SQLITE_STATIC);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
+      
+      ppgs.update();
+    }
   }
   
   // Get nouns/adjectives/adverbs from WordNet
@@ -342,110 +475,1046 @@ int main(int argc, char** argv)
     wnpref += '/';
   }
   
-  std::cout << "Reading words from WordNet..." << std::endl;
-  std::ifstream wnsfile(wnpref + "wn_s.pl");
-  if (!wnsfile.is_open())
+  // s table
   {
-    std::cout << "Invalid WordNet data directory." << std::endl;
-    print_usage();
+    std::ifstream wnsfile(wnpref + "wn_s.pl");
+    if (!wnsfile.is_open())
+    {
+      std::cout << "Invalid WordNet data directory." << std::endl;
+      print_usage();
+    }
+  
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnsfile, line))
+      {
+        break;
+      }
+    
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
+    
+    progress ppgs("Writing nouns, adjectives, and adverbs...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
+      
+      std::regex relation("^s\\(([134]\\d{8}),(\\d+),'([\\w ]+)',");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+    
+      int synset_id = stoi(relation_data[1]);
+      int wnum = stoi(relation_data[2]);
+      std::string word = relation_data[3];
+    
+      std::string query;
+      switch (synset_id / 100000000)
+      {
+        case 1: // Noun
+        {
+          if (nouns.count(word) == 1)
+          {
+            query = "INSERT INTO nouns (singular, plural) VALUES (?, ?)";
+          } else {
+            query = "INSERT INTO nouns (singular) VALUES (?)";
+          }
+        
+          break;
+        }
+      
+        case 2: // Verb
+        {
+          // Ignore
+        
+          break;
+        }
+      
+        case 3: // Adjective
+        {
+          if (adjectives.count(word) == 1)
+          {
+            query = "INSERT INTO adjectives (base_form, comparative, superlative) VALUES (?, ?, ?)";
+          } else {
+            query = "INSERT INTO adjectives (base_form) VALUES (?)";
+          }
+        
+          break;
+        }
+      
+        case 4: // Adverb
+        {
+          if (adjectives.count(word) == 1)
+          {
+            query = "INSERT INTO adverbs (base_form, comparative, superlative) VALUES (?, ?, ?)";
+          } else {
+            query = "INSERT INTO adverbs (base_form) VALUES (?)";
+          }
+        
+          break;
+        }
+      }
+    
+      sqlite3_stmt* ppstmt;
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
+    
+      sqlite3_bind_text(ppstmt, 1, word.c_str(), word.length(), SQLITE_STATIC);
+      switch (synset_id / 100000000)
+      {
+        case 1: // Noun
+        {
+          if (nouns.count(word) == 1)
+          {
+            sqlite3_bind_text(ppstmt, 2, nouns[word].plural.c_str(), nouns[word].plural.length(), SQLITE_STATIC);
+          }
+          
+          break;
+        }
+        
+        case 3: // Adjective
+        case 4: // Adverb
+        {
+          if (adjectives.count(word) == 1)
+          {
+            sqlite3_bind_text(ppstmt, 2, adjectives[word].comparative.c_str(), adjectives[word].comparative.length(), SQLITE_STATIC);
+            sqlite3_bind_text(ppstmt, 3, adjectives[word].superlative.c_str(), adjectives[word].superlative.length(), SQLITE_STATIC);
+          }
+          
+          break;
+        }
+      }
+    
+      if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      {
+        db_error(ppdb, query);
+      }
+    
+      sqlite3_finalize(ppstmt);
+    
+      query = "SELECT last_insert_rowid()";
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
+    
+      if (sqlite3_step(ppstmt) != SQLITE_ROW)
+      {
+        db_error(ppdb, query);
+      }
+    
+      int rowid = sqlite3_column_int(ppstmt, 0);
+      wn[synset_id][wnum] = rowid;
+    
+      sqlite3_finalize(ppstmt);
+      
+      std::string canonical(word);
+      std::transform(std::begin(canonical), std::end(canonical), std::begin(canonical), ::tolower);
+      if (pronunciations.count(canonical) == 1)
+      {
+        for (auto pronunciation : pronunciations[canonical])
+        {
+          switch (synset_id / 100000000)
+          {
+            case 1: // Noun
+            {
+              query = "INSERT INTO noun_pronunciations (noun_id, pronunciation) VALUES (?, ?)";
+              
+              break;
+            }
+            
+            case 3: // Adjective
+            {
+              query = "INSERT INTO adjective_pronunciations (adjective_id, pronunciation) VALUES (?, ?)";
+              
+              break;
+            }
+            
+            case 4: // Adverb
+            {
+              query = "INSERT INTO adverb_pronunciations (adverb_id, pronunciation) VALUES (?, ?)";
+              
+              break;
+            }
+          }
+
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+        
+          sqlite3_bind_int(ppstmt, 1, rowid);
+          sqlite3_bind_text(ppstmt, 2, pronunciation.c_str(), pronunciation.length(), SQLITE_STATIC);
+        
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+        
+          sqlite3_finalize(ppstmt);
+        }
+      }
+    }
   }
   
-  for (;;)
+  // While we're working on s
   {
-    std::string line;
-    if (!getline(wnsfile, line))
+    progress ppgs("Writing word synonyms...", wn.size());
+    for (auto sense : wn)
     {
-      break;
+      ppgs.update();
+      
+      for (auto word1 : sense.second)
+      {
+        for (auto word2 : sense.second)
+        {
+          if (word1 != word2)
+          {
+            std::string query;
+            switch (sense.first / 100000000)
+            {
+              case 1: // Noun
+              {
+                query = "INSERT INTO noun_synonymy (noun_1_id, noun_2_id) VALUES (?, ?)";
+        
+                break;
+              }
+      
+              case 2: // Verb
+              {
+                // Ignore
+        
+                break;
+              }
+      
+              case 3: // Adjective
+              {
+                query = "INSERT INTO adjective_synonymy (adjective_1_id, adjective_2_id) VALUES (?, ?)";
+        
+                break;
+              }
+      
+              case 4: // Adverb
+              {
+                query = "INSERT INTO adverb_synonymy (adverb_1_id, adverb_2_id) VALUES (?, ?)";
+        
+                break;
+              }
+            }
+            
+            sqlite3_stmt* ppstmt;
+            if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+            {
+              db_error(ppdb, query);
+            }
+    
+            sqlite3_bind_int(ppstmt, 1, word1.second);
+            sqlite3_bind_int(ppstmt, 2, word2.second);
+    
+            if (sqlite3_step(ppstmt) != SQLITE_DONE)
+            {
+              db_error(ppdb, query);
+            }
+    
+            sqlite3_finalize(ppstmt);
+          }
+        }
+      }
+    }
+  }
+  
+  // ant table
+  {
+    std::ifstream wnantfile(wnpref + "wn_ant.pl");
+    if (!wnantfile.is_open())
+    {
+      std::cout << "Invalid WordNet data directory." << std::endl;
+      print_usage();
     }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnantfile, line))
+      {
+        break;
+      }
     
-    if (line.back() == '\r')
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
+    
+    progress ppgs("Writing antonyms...", lines.size());
+    for (auto line : lines)
     {
-      line.pop_back();
+      ppgs.update();
+      
+      std::regex relation("^ant\\(([134]\\d{8}),(\\d+),([134]\\d{8}),(\\d+)\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+    
+      int synset_id_1 = stoi(relation_data[1]);
+      int wnum_1 = stoi(relation_data[2]);
+      int synset_id_2 = stoi(relation_data[3]);
+      int wnum_2 = stoi(relation_data[4]);
+    
+      std::string query;
+      switch (synset_id_1 / 100000000)
+      {
+        case 1: // Noun
+        {
+          query = "INSERT INTO noun_antonymy (noun_1_id, noun_2_id) VALUES (?, ?)";
+        
+          break;
+        }
+      
+        case 2: // Verb
+        {
+          // Ignore
+        
+          break;
+        }
+      
+        case 3: // Adjective
+        {
+          query = "INSERT INTO adjective_antonymy (adjective_1_id, adjective_2_id) VALUES (?, ?)";
+        
+          break;
+        }
+      
+        case 4: // Adverb
+        {
+          query = "INSERT INTO adverb_antonymy (adverb_1_id, adverb_2_id) VALUES (?, ?)";
+        
+          break;
+        }
+      }
+    
+      sqlite3_stmt* ppstmt;
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
+    
+      sqlite3_bind_int(ppstmt, 1, wn[synset_id_1][wnum_1]);
+      sqlite3_bind_int(ppstmt, 2, wn[synset_id_2][wnum_2]);
+    
+      if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      {
+        db_error(ppdb, query);
+      }
+    
+      sqlite3_finalize(ppstmt);
+    }
+  }
+  
+  // at table
+  {
+    std::ifstream wnatfile(wnpref + "wn_at.pl");
+    if (!wnatfile.is_open())
+    {
+      std::cout << "Invalid WordNet data directory." << std::endl;
+      print_usage();
     }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnatfile, line))
+      {
+        break;
+      }
     
-    std::regex relation("^s\\(([134]\\d{8}),(\\d+),'([\\w ]+)',");
-    std::smatch relation_data;
-    if (!std::regex_search(line, relation_data, relation))
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
+    
+    progress ppgs("Writing variations...", lines.size());
+    for (auto line : lines)
     {
-      continue;
+      ppgs.update();
+      
+      std::regex relation("^at\\((1\\d{8}),(3\\d{8})\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int synset_id_2 = stoi(relation_data[2]);
+      std::string query("INSERT INTO variation (noun_id, adjective_id) VALUES (?, ?)");
+      
+      for (auto mapping1 : wn[synset_id_1])
+      {
+        for (auto mapping2 : wn[synset_id_2])
+        {
+          sqlite3_stmt* ppstmt;
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, mapping1.second);
+          sqlite3_bind_int(ppstmt, 2, mapping2.second);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
     }
+  }
+  
+  // hyp table
+  {
+    std::ifstream wnhypfile(wnpref + "wn_hyp.pl");
+    if (!wnhypfile.is_open())
+    {
+      std::cout << "Invalid WordNet data directory." << std::endl;
+      print_usage();
+    }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnhypfile, line))
+      {
+        break;
+      }
     
-    int synset_id = stoi(relation_data[1]);
-    int wnum = stoi(relation_data[2]);
-    std::string word = relation_data[3];
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
     
-    std::string query;
-    switch (synset_id / 100000000)
+    progress ppgs("Writing hypernyms...", lines.size());
+    for (auto line : lines)
     {
-      case 1: // Noun
+      ppgs.update();
+      
+      std::regex relation("^hyp\\((1\\d{8}),(1\\d{8})\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int synset_id_2 = stoi(relation_data[2]);
+      std::string query("INSERT INTO hypernymy (hyponym_id, hypernym_id) VALUES (?, ?)");
+      
+      for (auto mapping1 : wn[synset_id_1])
+      {
+        for (auto mapping2 : wn[synset_id_2])
+        {
+          sqlite3_stmt* ppstmt;
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, mapping1.second);
+          sqlite3_bind_int(ppstmt, 2, mapping2.second);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
+    }
+  }
+  
+  // ins table
+  {
+    std::ifstream wninsfile(wnpref + "wn_ins.pl");
+    if (!wninsfile.is_open())
+    {
+      std::cout << "Invalid WordNet data directory." << std::endl;
+      print_usage();
+    }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wninsfile, line))
       {
-        query = "INSERT INTO nouns (form) VALUES (?)";
-        
         break;
       }
+    
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
       
-      case 2: // Verb
+      lines.push_back(line);
+    }
+    
+    progress ppgs("Writing instantiations...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
+      
+      std::regex relation("^ins\\((1\\d{8}),(1\\d{8})\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int synset_id_2 = stoi(relation_data[2]);
+      std::string query("INSERT INTO instantiation (instance_id, class_id) VALUES (?, ?)");
+      
+      for (auto mapping1 : wn[synset_id_1])
+      {
+        for (auto mapping2 : wn[synset_id_2])
+        {
+          sqlite3_stmt* ppstmt;
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, mapping1.second);
+          sqlite3_bind_int(ppstmt, 2, mapping2.second);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
+    }
+  }
+  
+  // mm table
+  {
+    std::ifstream wnmmfile(wnpref + "wn_mm.pl");
+    if (!wnmmfile.is_open())
+    {
+      std::cout << "Invalid WordNet data directory." << std::endl;
+      print_usage();
+    }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnmmfile, line))
       {
-        // Ignore
-        
         break;
       }
+    
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
+    
+    progress ppgs("Writing member meronyms...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
       
-      case 3: // Adjective
+      std::regex relation("^mm\\((1\\d{8}),(1\\d{8})\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int synset_id_2 = stoi(relation_data[2]);
+      std::string query("INSERT INTO member_meronymy (holonym_id, meronym_id) VALUES (?, ?)");
+      
+      for (auto mapping1 : wn[synset_id_1])
+      {
+        for (auto mapping2 : wn[synset_id_2])
+        {
+          sqlite3_stmt* ppstmt;
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, mapping1.second);
+          sqlite3_bind_int(ppstmt, 2, mapping2.second);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
+    }
+  }
+  
+  // ms table
+  {
+    std::ifstream wnmsfile(wnpref + "wn_ms.pl");
+    if (!wnmsfile.is_open())
+    {
+      std::cout << "Invalid WordNet data directory." << std::endl;
+      print_usage();
+    }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnmsfile, line))
       {
-        query = "INSERT INTO adjectives (form) VALUES (?)";
-        
         break;
       }
+    
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
       
-      case 4: // Adverb
+      lines.push_back(line);
+    }
+    
+    progress ppgs("Writing substance meronyms...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
+      
+      std::regex relation("^ms\\((1\\d{8}),(1\\d{8})\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int synset_id_2 = stoi(relation_data[2]);
+      std::string query("INSERT INTO substance_meronymy (holonym_id, meronym_id) VALUES (?, ?)");
+      
+      for (auto mapping1 : wn[synset_id_1])
+      {
+        for (auto mapping2 : wn[synset_id_2])
+        {
+          sqlite3_stmt* ppstmt;
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, mapping1.second);
+          sqlite3_bind_int(ppstmt, 2, mapping2.second);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
+    }
+  }
+  
+  // mm table
+  {
+    std::ifstream wnmpfile(wnpref + "wn_mp.pl");
+    if (!wnmpfile.is_open())
+    {
+      std::cout << "Invalid WordNet data directory." << std::endl;
+      print_usage();
+    }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnmpfile, line))
       {
-        query = "INSERT INTO adverbs (form) VALUES (?)";
-        
         break;
       }
+    
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
     }
     
-    sqlite3_stmt* ppstmt;
-    if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+    progress ppgs("Writing part meronyms...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
+      
+      std::regex relation("^mp\\((1\\d{8}),(1\\d{8})\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int synset_id_2 = stoi(relation_data[2]);
+      std::string query("INSERT INTO part_meronymy (holonym_id, meronym_id) VALUES (?, ?)");
+      
+      for (auto mapping1 : wn[synset_id_1])
+      {
+        for (auto mapping2 : wn[synset_id_2])
+        {
+          sqlite3_stmt* ppstmt;
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, mapping1.second);
+          sqlite3_bind_int(ppstmt, 2, mapping2.second);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
+    }
+  }
+  
+  // per table
+  {
+    std::ifstream wnperfile(wnpref + "wn_per.pl");
+    if (!wnperfile.is_open())
     {
-      std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
-      sqlite3_close_v2(ppdb);
+      std::cout << "Invalid WordNet data directory." << std::endl;
       print_usage();
     }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnperfile, line))
+      {
+        break;
+      }
+    
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
+    
+    progress ppgs("Writing pertainyms and mannernyms...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
+      
+      std::regex relation("^per\\(([34]\\d{8}),(\\d+),([13]\\d{8}),(\\d+)\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int wnum_1 = stoi(relation_data[2]);
+      int synset_id_2 = stoi(relation_data[3]);
+      int wnum_2 = stoi(relation_data[4]);
+      std::string query;
+      switch (synset_id_1 / 100000000)
+      {
+        case 3: // Adjective
+        {
+          // This is a pertainym, the second word should be a noun
+          // Technically it can be an adjective but we're ignoring that
+          if (synset_id_2 / 100000000 != 1)
+          {
+            continue;
+          }
+          
+          query = "INSERT INTO pertainymy (pertainym_id, noun_id) VALUES (?, ?)";
+          
+          break;
+        }
+        
+        case 4: // Adverb
+        {
+          // This is a mannernym, the second word should be an adjective
+          if (synset_id_2 / 100000000 != 3)
+          {
+            continue;
+          }
+          
+          query = "INSERT INTO mannernymy (mannernym_id, adjective_id) VALUES (?, ?)";
+          
+          break;
+        }
+      }
+      
+      sqlite3_stmt* ppstmt;
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
     
-    sqlite3_bind_text(ppstmt, 1, word.c_str(), word.length(), SQLITE_STATIC);
+      sqlite3_bind_int(ppstmt, 1, wn[synset_id_1][wnum_1]);
+      sqlite3_bind_int(ppstmt, 2, wn[synset_id_2][wnum_2]);
     
-    if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      {
+        db_error(ppdb, query);
+      }
+    
+      sqlite3_finalize(ppstmt);
+    }
+  }
+  
+  // sa table
+  {
+    std::ifstream wnsafile(wnpref + "wn_sa.pl");
+    if (!wnsafile.is_open())
     {
-      std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
-      sqlite3_close_v2(ppdb);
+      std::cout << "Invalid WordNet data directory." << std::endl;
       print_usage();
     }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnsafile, line))
+      {
+        break;
+      }
+    
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
     
-    sqlite3_finalize(ppstmt);
+    progress ppgs("Writing specifications...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
+      
+      std::regex relation("^per\\((3\\d{8}),(\\d+),(3\\d{8}),(\\d+)\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int wnum_1 = stoi(relation_data[2]);
+      int synset_id_2 = stoi(relation_data[3]);
+      int wnum_2 = stoi(relation_data[4]);
+      std::string query("INSERT INTO specification (general_id, specific_id) VALUES (?, ?)");
+      
+      sqlite3_stmt* ppstmt;
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
+    
+      sqlite3_bind_int(ppstmt, 1, wn[synset_id_1][wnum_1]);
+      sqlite3_bind_int(ppstmt, 2, wn[synset_id_2][wnum_2]);
     
-    query = "SELECT last_insert_rowid()";
-    if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      {
+        db_error(ppdb, query);
+      }
+    
+      sqlite3_finalize(ppstmt);
+    }
+  }
+  /*
+  // sim table
+  {
+    std::ifstream wnsimfile(wnpref + "wn_sim.pl");
+    if (!wnsimfile.is_open())
     {
-      std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
-      sqlite3_close_v2(ppdb);
+      std::cout << "Invalid WordNet data directory." << std::endl;
       print_usage();
     }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnsimfile, line))
+      {
+        break;
+      }
+    
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
     
-    if (sqlite3_step(ppstmt) != SQLITE_ROW)
+    progress ppgs("Writing sense synonyms...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
+      
+      std::regex relation("^sim\\((3\\d{8}),(3\\d{8})\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id_1 = stoi(relation_data[1]);
+      int synset_id_2 = stoi(relation_data[2]);
+      std::string query("INSERT INTO adjective_synonymy (adjective_1_id, adjective_2_id) VALUES (?, ?)");
+      
+      for (auto mapping1 : wn[synset_id_1])
+      {
+        for (auto mapping2 : wn[synset_id_2])
+        {
+          sqlite3_stmt* ppstmt;
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, mapping1.second);
+          sqlite3_bind_int(ppstmt, 2, mapping2.second);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_reset(ppstmt);
+          sqlite3_clear_bindings(ppstmt);
+          
+          sqlite3_bind_int(ppstmt, 1, mapping2.second);
+          sqlite3_bind_int(ppstmt, 2, mapping1.second);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
+    }
+  }
+  */
+  // syntax table
+  {
+    std::ifstream wnsyntaxfile(wnpref + "wn_syntax.pl");
+    if (!wnsyntaxfile.is_open())
     {
-      std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
-      sqlite3_close_v2(ppdb);
+      std::cout << "Invalid WordNet data directory." << std::endl;
       print_usage();
     }
+
+    std::list<std::string> lines;
+    for (;;)
+    {
+      std::string line;
+      if (!getline(wnsyntaxfile, line))
+      {
+        break;
+      }
     
-    wn[synset_id][wnum] = sqlite3_column_int(ppstmt, 0);
+      if (line.back() == '\r')
+      {
+        line.pop_back();
+      }
+      
+      lines.push_back(line);
+    }
     
-    sqlite3_finalize(ppstmt);
+    progress ppgs("Writing adjective syntax markers...", lines.size());
+    for (auto line : lines)
+    {
+      ppgs.update();
+      
+      std::regex relation("^syntax\\((3\\d{8}),(\\d+),([ipa])p?\\)\\.");
+      std::smatch relation_data;
+      if (!std::regex_search(line, relation_data, relation))
+      {
+        continue;
+      }
+      
+      int synset_id = stoi(relation_data[1]);
+      int wnum = stoi(relation_data[2]);
+      std::string syn = relation_data[3];
+      std::string query("UPDATE adjectives SET position = ? WHERE adjective_id = ?");
+      
+      sqlite3_stmt* ppstmt;
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
+      
+      sqlite3_bind_text(ppstmt, 1, syn.c_str(), 1, SQLITE_STATIC);
+      sqlite3_bind_int(ppstmt, 2, wn[synset_id][wnum]);
+      
+      if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      {
+        db_error(ppdb, query);
+      }
+      
+      sqlite3_finalize(ppstmt);
+    }
   }
   
   sqlite3_close_v2(ppdb);
   
   std::cout << "Done." << std::endl;
-}
\ No newline at end of file
+}