Added verb frames

In addition:
- Added prepositions.
- Rewrote a lot of the query interface. It now, for a lot of relationships, supports nested AND, OR, and NOT logic.
- Rewrote the token class. It is now a union-like class instead of being polymorphic, which means smart pointers are no longer necessary.
- Querying with regards to word derivation has been temporarily removed.
- Sentinel values are now supported for all word types.
- The VerbNet data retrieved from http://verbs.colorado.edu/~mpalmer/projects/verbnet/downloads.html was found to not be perfectly satisfactory in some regards, especially regarding adjective phrases. A patch file is now included in the repository describing the changes made to the VerbNet v3.2 download for the canonical verbly datafile.

5 files changed, 1069 insertions, 25 deletions

@@ -5,7 +5,7 @@ find_package(PkgConfig)
 pkg_check_modules(sqlite3 sqlite3 REQUIRED)
 find_package(libxml2 REQUIRED)
 
-include_directories(${sqlite3_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR})
+include_directories(${sqlite3_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ../vendor/json/src)
 add_executable(generator generator.cpp)
 set_property(TARGET generator PROPERTY CXX_STANDARD 11)
 set_property(TARGET generator PROPERTY CXX_STANDARD_REQUIRED ON)
@@ -11,36 +11,75 @@
 #include <regex>
 #include <list>
 #include <algorithm>
+#include <json.hpp>
 #include "progress.h"
+#include "../lib/util.h"
 
-struct verb {
+using json = nlohmann::json;
+
+struct verb_t {
   std::string infinitive;
   std::string past_tense;
   std::string past_participle;
   std::string ing_form;
   std::string s_form;
+  int id;
 };
 
-struct adjective {
+struct adjective_t {
   std::string base;
   std::string comparative;
   std::string superlative;
 };
 
-struct noun {
+struct noun_t {
   std::string singular;
   std::string plural;
 };
 
-struct group {
+struct selrestr_t {
+  enum class type_t {
+    singleton,
+    andlogic,
+    orlogic,
+    empty
+  };
+  type_t type;
+  std::string restriction;
+  bool pos;
+  std::list<selrestr_t> subordinates;
+};
+
+struct framepart_t {
+  enum class type_t {
+    np,
+    v,
+    pp,
+    adj,
+    adv,
+    lex
+  };
+  type_t type;
+  std::string role;
+  selrestr_t selrestrs;
+  std::set<std::string> preprestrs;
+  std::set<std::string> synrestrs;
+  std::list<std::string> choices;
+  std::string lexval;
+};
+
+struct group_t {
   std::string id;
+  std::string parent;
   std::set<std::string> members;
+  std::map<std::string, selrestr_t> roles;
+  std::list<std::list<framepart_t>> frames;
 };
 
-std::map<std::string, group> groups;
-std::map<std::string, verb> verbs;
-std::map<std::string, adjective> adjectives;
-std::map<std::string, noun> nouns;
+std::map<std::string, group_t> groups;
+std::map<std::string, verb_t> verbs;
+std::map<std::string, adjective_t> adjectives;
+std::map<std::string, noun_t> nouns;
 std::map<int, std::map<int, int>> wn;
 std::map<std::string, std::set<std::string>> pronunciations;
 
@@ -59,15 +98,97 @@ void print_usage()
   exit(1);
 }
 
-void db_error(sqlite3* ppdb, std::string)
+void db_error(sqlite3* ppdb, std::string query)
 {
   std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl;
+  std::cout << query << std::endl;
   sqlite3_close_v2(ppdb);
   print_usage();
 }
 
-/*
-void parse_group(xmlNodePtr top, std::string filename)
+json export_selrestrs(selrestr_t r)
+{
+  if (r.type == selrestr_t::type_t::empty)
+  {
+    return {};
+  } else if (r.type == selrestr_t::type_t::singleton)
+  {
+    json result;
+    result["type"] = r.restriction;
+    result["pos"] = r.pos;
+    return result;
+  } else {
+    json result;
+    if (r.type == selrestr_t::type_t::andlogic)
+    {
+      result["logic"] = "and";
+    } else {
+      result["logic"] = "or";
+    }
+    
+    std::list<json> outlist;
+    std::transform(std::begin(r.subordinates), std::end(r.subordinates), std::back_inserter(outlist), &export_selrestrs);
+    result["children"] = outlist;
+    
+    return result;
+  }
+}
+
+selrestr_t parse_selrestrs(xmlNodePtr top, std::string filename)
+{
+  selrestr_t r;
+  xmlChar* key;
+  
+  if (!xmlStrcmp(top->name, (const xmlChar*) "SELRESTRS"))
+  {
+    if (xmlChildElementCount(top) == 0)
+    {
+      r.type = selrestr_t::type_t::empty;
+    } else if (xmlChildElementCount(top) == 1)
+    {
+      r = parse_selrestrs(xmlFirstElementChild(top), filename);
+    } else {
+      r.type = selrestr_t::type_t::andlogic;
+      
+      if (xmlHasProp(top, (const xmlChar*) "logic"))
+      {
+        key = xmlGetProp(top, (const xmlChar*) "logic");
+        if (!xmlStrcmp(key, (const xmlChar*) "or"))
+        {
+          r.type = selrestr_t::type_t::orlogic;
+        }
+        xmlFree(key);
+      }
+  
+      for (xmlNodePtr selrestr = top->xmlChildrenNode; selrestr != nullptr; selrestr = selrestr->next)
+      {
+        if (!xmlStrcmp(selrestr->name, (const xmlChar*) "SELRESTRS") || !xmlStrcmp(selrestr->name, (const xmlChar*) "SELRESTR"))
+        {
+          r.subordinates.push_back(parse_selrestrs(selrestr, filename));
+        }
+      }
+    }
+  } else if (!xmlStrcmp(top->name, (const xmlChar*) "SELRESTR"))
+  {
+    r.type = selrestr_t::type_t::singleton;
+    
+    key = xmlGetProp(top, (xmlChar*) "Value");
+    r.pos = (std::string((const char*)key) == "+");
+    xmlFree(key);
+
+    key = xmlGetProp(top, (xmlChar*) "type");
+    r.restriction = (const char*) key;
+    xmlFree(key);
+  } else {
+    // Invalid
+    std::cout << "Bad VerbNet file format: " << filename << std::endl;
+    print_usage();
+  }
+  
+  return r;
+}
+
+group_t& parse_group(xmlNodePtr top, std::string filename)
 {
   xmlChar* key = xmlGetProp(top, (xmlChar*) "ID");
   if (key == 0)
@@ -75,41 +196,183 @@ void parse_group(xmlNodePtr top, std::string filename)
     std::cout << "Bad VerbNet file format: " << filename << std::endl;
     print_usage();
   }
-  std::string vnid = key;
+  std::string vnid = (const char*)key;
   vnid = vnid.substr(vnid.find_first_of("-")+1);
   xmlFree(key);
   
-  group g;
+  group_t g;
   g.id = vnid;
   
   for (xmlNodePtr node = top->xmlChildrenNode; node != nullptr; node = node->next)
   {
-    if (!xmlStrcmp(node->name, (const xmlChar*) "MEMBERS"))
+    if (!xmlStrcmp(node->name, (const xmlChar*) "SUBCLASSES"))
+    {
+      for (xmlNodePtr subclass = node->xmlChildrenNode; subclass != nullptr; subclass = subclass->next)
+      {
+        if (!xmlStrcmp(subclass->name, (const xmlChar*) "VNSUBCLASS"))
+        {
+          auto& sg = parse_group(subclass, filename);
+          sg.parent = vnid;
+          
+          for (auto member : sg.members)
+          {
+            g.members.insert(member);
+          }
+          
+          // The schema requires that subclasses appear after role definitions, so we can do this now
+          for (auto role : g.roles)
+          {
+            if (sg.roles.count(role.first) == 0)
+            {
+              sg.roles[role.first] = role.second;
+            }
+          }
+        }
+      }
+    } else if (!xmlStrcmp(node->name, (const xmlChar*) "MEMBERS"))
     {
       for (xmlNodePtr member = node->xmlChildrenNode; member != nullptr; member = member->next)
       {
         if (!xmlStrcmp(member->name, (const xmlChar*) "MEMBER"))
         {
           key = xmlGetProp(member, (xmlChar*) "name");
-          g.members.insert(key);
+          g.members.insert((const char*)key);
           xmlFree(key);
         }
       }
+    } else if (!xmlStrcmp(node->name, (const xmlChar*) "THEMROLES"))
+    {
+      for (xmlNodePtr role = node->xmlChildrenNode; role != nullptr; role = role->next)
+      {
+        if (!xmlStrcmp(role->name, (const xmlChar*) "THEMROLE"))
+        {
+          selrestr_t r;
+          r.type = selrestr_t::type_t::empty;
+          
+          key = xmlGetProp(role, (const xmlChar*) "type");
+          std::string type = (const char*)key;
+          xmlFree(key);
+          
+          for (xmlNodePtr rolenode = role->xmlChildrenNode; rolenode != nullptr; rolenode = rolenode->next)
+          {
+            if (!xmlStrcmp(rolenode->name, (const xmlChar*) "SELRESTRS"))
+            {
+              r = parse_selrestrs(rolenode, filename);
+            }
+          }
+          
+          g.roles[type] = r;
+        }
+      }
     } else if (!xmlStrcmp(node->name, (const xmlChar*) "FRAMES"))
     {
       for (xmlNodePtr frame = node->xmlChildrenNode; frame != nullptr; frame = frame->next)
       {
         if (!xmlStrcmp(frame->name, (const xmlChar*) "FRAME"))
         {
+          std::list<framepart_t> f;
+          
           for (xmlNodePtr framenode = frame->xmlChildrenNode; framenode != nullptr; framenode = framenode->next)
           {
-            
+            if (!xmlStrcmp(framenode->name, (const xmlChar*) "SYNTAX"))
+            {
+              for (xmlNodePtr syntaxnode = framenode->xmlChildrenNode; syntaxnode != nullptr; syntaxnode = syntaxnode->next)
+              {
+                framepart_t fp;
+                
+                if (!xmlStrcmp(syntaxnode->name, (const xmlChar*) "NP"))
+                {
+                  fp.type = framepart_t::type_t::np;
+                  
+                  key = xmlGetProp(syntaxnode, (xmlChar*) "value");
+                  fp.role = (const char*)key;
+                  xmlFree(key);
+                  
+                  fp.selrestrs.type = selrestr_t::type_t::empty;
+                  
+                  for (xmlNodePtr npnode = syntaxnode->xmlChildrenNode; npnode != nullptr; npnode = npnode->next)
+                  {
+                    if (!xmlStrcmp(npnode->name, (const xmlChar*) "SYNRESTRS"))
+                    {
+                      for (xmlNodePtr synrestr = npnode->xmlChildrenNode; synrestr != nullptr; synrestr = synrestr->next)
+                      {
+                        if (!xmlStrcmp(synrestr->name, (const xmlChar*) "SYNRESTR"))
+                        {
+                          key = xmlGetProp(synrestr, (xmlChar*) "type");
+                          fp.synrestrs.insert(std::string((const char*)key));
+                          xmlFree(key);
+                        }
+                      }
+                    }
+                  
+                    if (!xmlStrcmp(npnode->name, (const xmlChar*) "SELRESTRS"))
+                    {
+                      fp.selrestrs = parse_selrestrs(npnode, filename);
+                    }
+                  }
+                } else if (!xmlStrcmp(syntaxnode->name, (xmlChar*) "VERB"))
+                {
+                  fp.type = framepart_t::type_t::v;
+                } else if (!xmlStrcmp(syntaxnode->name, (xmlChar*) "PREP"))
+                {
+                  fp.type = framepart_t::type_t::pp;
+                  
+                  if (xmlHasProp(syntaxnode, (xmlChar*) "value"))
+                  {
+                    key = xmlGetProp(syntaxnode, (xmlChar*) "value");
+                    std::string choices = (const char*)key;
+                    xmlFree(key);
+                  
+                    fp.choices = verbly::split<std::list<std::string>>(choices, " ");
+                  }
+                  
+                  for (xmlNodePtr npnode = syntaxnode->xmlChildrenNode; npnode != nullptr; npnode = npnode->next)
+                  {
+                    if (!xmlStrcmp(npnode->name, (const xmlChar*) "SELRESTRS"))
+                    {
+                      for (xmlNodePtr synrestr = npnode->xmlChildrenNode; synrestr != nullptr; synrestr = synrestr->next)
+                      {
+                        if (!xmlStrcmp(synrestr->name, (const xmlChar*) "SELRESTR"))
+                        {
+                          key = xmlGetProp(synrestr, (xmlChar*) "type");
+                          fp.preprestrs.insert(std::string((const char*)key));
+                          xmlFree(key);
+                        }
+                      }
+                    }
+                  }
+                } else if (!xmlStrcmp(syntaxnode->name, (xmlChar*) "ADJ"))
+                {
+                  fp.type = framepart_t::type_t::adj;
+                } else if (!xmlStrcmp(syntaxnode->name, (xmlChar*) "ADV"))
+                {
+                  fp.type = framepart_t::type_t::adv;
+                } else if (!xmlStrcmp(syntaxnode->name, (xmlChar*) "LEX"))
+                {
+                  fp.type = framepart_t::type_t::lex;
+                  
+                  key = xmlGetProp(syntaxnode, (xmlChar*) "value");
+                  fp.lexval = (const char*)key;
+                  xmlFree(key);
+                } else {
+                  continue;
+                }
+                
+                f.push_back(fp);
+              }
+              
+              g.frames.push_back(f);
+            }
           }
         }
       }
     }
   }
-}*/
+  
+  groups[vnid] = g;
+  
+  return groups[vnid];
+}
 
 int main(int argc, char** argv)
 {
@@ -118,7 +381,10 @@ int main(int argc, char** argv)
     print_usage();
   }
   
-  /*DIR* dir;
+  // VerbNet data
+  std::cout << "Reading verb frames..." << std::endl;
+  
+  DIR* dir;
   if ((dir = opendir(argv[1])) == nullptr)
   {
     std::cout << "Invalid VerbNet data directory." << std::endl;
@@ -160,7 +426,7 @@ int main(int argc, char** argv)
     parse_group(top, filename);
   }
   
-  closedir(dir);*/
+  closedir(dir);
   
   // Get verbs from AGID
   std::cout << "Reading inflections..." << std::endl;
@@ -222,7 +488,7 @@ int main(int argc, char** argv)
     {
       case 'V':
       {
-        verb v;
+        verb_t v;
         v.infinitive = word;
         if (forms.size() == 4)
         {
@@ -258,7 +524,7 @@ int main(int argc, char** argv)
       
       case 'A':
       {
-        adjective adj;
+        adjective_t adj;
         adj.base = word;
         if (forms.size() == 2)
         {
@@ -276,7 +542,7 @@ int main(int argc, char** argv)
       
       case 'N':
       {
-        noun n;
+        noun_t n;
         n.singular = word;
         if (forms.size() == 1)
         {
@@ -388,6 +654,85 @@ int main(int argc, char** argv)
     sqlite3_finalize(schmstmt);
   }
   
+  std::cout << "Writing prepositions..." << std::endl;
+  std::ifstream prepfile("prepositions.txt");
+  if (!prepfile.is_open())
+  {
+    std::cout << "Could not find prepositions file" << std::endl;
+    print_usage();
+  }
+  
+  for (;;)
+  {
+    std::string line;
+    if (!getline(prepfile, line))
+    {
+      break;
+    }
+    
+    if (line.back() == '\r')
+    {
+      line.pop_back();
+    }
+    
+    std::regex relation("^([^:]+): (.+)");
+    std::smatch relation_data;
+    std::regex_search(line, relation_data, relation);
+    std::string prep = relation_data[1];
+    std::list<std::string> groups = verbly::split<std::list<std::string>>(relation_data[2], ", ");
+    
+    std::string query("INSERT INTO prepositions (form) VALUES (?)");
+    sqlite3_stmt* ppstmt;
+    
+    if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+    {
+      db_error(ppdb, query);
+    }
+    
+    sqlite3_bind_text(ppstmt, 1, prep.c_str(), prep.length(), SQLITE_STATIC);
+    
+    if (sqlite3_step(ppstmt) != SQLITE_DONE)
+    {
+      db_error(ppdb, query);
+    }
+    
+    sqlite3_finalize(ppstmt);
+    
+    query = "SELECT last_insert_rowid()";
+    if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+    {
+      db_error(ppdb, query);
+    }
+    
+    if (sqlite3_step(ppstmt) != SQLITE_ROW)
+    {
+      db_error(ppdb, query);
+    }
+    
+    int rowid = sqlite3_column_int(ppstmt, 0);
+    sqlite3_finalize(ppstmt);
+    
+    for (auto group : groups)
+    {
+      query = "INSERT INTO preposition_groups (preposition_id, groupname) VALUES (?, ?)";
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
+      
+      sqlite3_bind_int(ppstmt, 1, rowid);
+      sqlite3_bind_text(ppstmt, 2, group.c_str(), group.length(), SQLITE_STATIC);
+      
+      if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      {
+        db_error(ppdb, query);
+      }
+      
+      sqlite3_finalize(ppstmt);
+    }
+  }
+  
+  
   {
     progress ppgs("Writing verbs...", verbs.size());
     for (auto& mapping : verbs)
@@ -431,6 +776,8 @@ int main(int argc, char** argv)
     
         sqlite3_finalize(ppstmt);
         
+        mapping.second.id = rowid;
+        
         for (auto pronunciation : pronunciations[canonical])
         {
           query = "INSERT INTO verb_pronunciations (verb_id, pronunciation) VALUES (?, ?)";
@@ -455,6 +802,160 @@ int main(int argc, char** argv)
     }
   }
   
+  {
+    progress ppgs("Writing verb frames...", groups.size());
+    for (auto& mapping : groups)
+    {
+      std::list<json> roledatal;
+      std::transform(std::begin(mapping.second.roles), std::end(mapping.second.roles), std::back_inserter(roledatal), [] (std::pair<std::string, selrestr_t> r) {
+        json role;
+        role["type"] = r.first;
+        role["selrestrs"] = export_selrestrs(r.second);
+        
+        return role;
+      });
+      
+      json roledata(roledatal);
+      std::string rdm = roledata.dump();
+      
+      sqlite3_stmt* ppstmt;
+      std::string query("INSERT INTO groups (data) VALUES (?)");
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
+      
+      sqlite3_bind_blob(ppstmt, 1, rdm.c_str(), rdm.size(), SQLITE_STATIC);
+      
+      if (sqlite3_step(ppstmt) != SQLITE_DONE)
+      {
+        db_error(ppdb, query);
+      }
+      
+      sqlite3_finalize(ppstmt);
+      
+      query = "SELECT last_insert_rowid()";
+      if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+      {
+        db_error(ppdb, query);
+      }
+      
+      if (sqlite3_step(ppstmt) != SQLITE_ROW)
+      {
+        db_error(ppdb, query);
+      }
+      
+      int gid = sqlite3_column_int(ppstmt, 0);
+      sqlite3_finalize(ppstmt);
+      
+      for (auto frame : mapping.second.frames)
+      {
+        std::list<json> fdatap;
+        std::transform(std::begin(frame), std::end(frame), std::back_inserter(fdatap), [] (framepart_t& fp) {
+          json part;
+          
+          switch (fp.type)
+          {
+            case framepart_t::type_t::np:
+            {
+              part["type"] = "np";
+              part["role"] = fp.role;
+              part["selrestrs"] = export_selrestrs(fp.selrestrs);
+              part["synrestrs"] = fp.synrestrs;
+              
+              break;
+            }
+            
+            case framepart_t::type_t::pp:
+            {
+              part["type"] = "pp";
+              part["values"] = fp.choices;
+              part["preprestrs"] = fp.preprestrs;
+              
+              break;
+            }
+            
+            case framepart_t::type_t::v:
+            {
+              part["type"] = "v";
+              
+              break;
+            }
+            
+            case framepart_t::type_t::adj:
+            {
+              part["type"] = "adj";
+              
+              break;
+            }
+            
+            case framepart_t::type_t::adv:
+            {
+              part["type"] = "adv";
+              
+              break;
+            }
+            
+            case framepart_t::type_t::lex:
+            {
+              part["type"] = "lex";
+              part["value"] = fp.lexval;
+              
+              break;
+            }
+          }
+          
+          return part;
+        });
+        
+        json fdata(fdatap);
+        std::string marshall = fdata.dump();
+        
+        query = "INSERT INTO frames (group_id, data) VALUES (?, ?)";
+        if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+        {
+          db_error(ppdb, query);
+        }
+        
+        sqlite3_bind_int(ppstmt, 1, gid);
+        sqlite3_bind_blob(ppstmt, 2, marshall.c_str(), marshall.length(), SQLITE_STATIC);
+        
+        if (sqlite3_step(ppstmt) != SQLITE_DONE)
+        {
+          db_error(ppdb, query);
+        }
+        
+        sqlite3_finalize(ppstmt);
+      }
+      
+      for (auto member : mapping.second.members)
+      {
+        if (verbs.count(member) == 1)
+        {
+          auto& v = verbs[member];
+          
+          query = "INSERT INTO verb_groups (verb_id, group_id) VALUES (?, ?)";
+          if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_bind_int(ppstmt, 1, v.id);
+          sqlite3_bind_int(ppstmt, 2, gid);
+          
+          if (sqlite3_step(ppstmt) != SQLITE_DONE)
+          {
+            db_error(ppdb, query);
+          }
+          
+          sqlite3_finalize(ppstmt);
+        }
+      }
+      
+      ppgs.update();
+    }
+  }
+  
   // Get nouns/adjectives/adverbs from WordNet
   // Useful relations:
   // - s: master list
@@ -0,0 +1,49 @@
+from: src, path, spatial, loc
+out: src, path, spatial
+out of: src, path, spatial, loc
+off: src, path, spatial, loc
+off of: src, path, spatial
+into: dest_conf, dest, path, spatial
+onto: dest_conf, dest, path, spatial
+for: dest_dir, dest, path, spatial
+at: dest_dir, dest, path, spatial, loc
+to: dest_dir, dest, path, spatial
+towards: dest_dir, dest, path, spatial
+along: dir, path, spatial, loc
+across: dir, path, spatial
+around: dir, path, spatial, loc
+down: dir, path, spatial
+over: dir, path, spatial, loc
+past: dir, path, spatial
+round: dir, path, spatial, loc
+through: dir, path, spatial
+towards: dir, path, spatial
+up: dir, path, spatial
+about: loc, spatial
+above: loc, spatial
+against: loc, spatial
+alongside: loc, spatial
+amid: loc, spatial
+among: loc, spatial
+amongst: loc, spatial
+astride: loc, spatial
+athwart: loc, spatial
+before: loc, spatial
+behind: loc, spatial
+beside: loc, spatial
+between: loc, spatial
+beyond: loc, spatial
+by: loc, spatial
+in: loc, spatial
+in front of: loc, spatial
+inside: loc, spatial
+near: loc, spatial
+next to: loc, spatial
+on: loc, spatial
+opposite: loc, spatial
+outside: loc, spatial
+throughout: loc, spatial
+under: loc, spatial
+underneath: loc, spatial
+upon: loc, spatial
+within: loc, spatial
\ No newline at end of file
@@ -11,8 +11,7 @@ CREATE TABLE `verbs` (
 DROP TABLE IF EXISTS `groups`;
 CREATE TABLE `groups` (
   `group_id` INTEGER PRIMARY KEY,
-  `parent_id` INTEGER,
-  FOREIGN KEY (`parent_id`) REFERENCES `groups`(`group_id`)
+  `data` BLOB NOT NULL
 );
 
 DROP TABLE IF EXISTS `frames`;
@@ -251,3 +250,16 @@ CREATE TABLE `adverb_adverb_derivation` (
   FOREIGN KEY (`adverb_1_id`) REFERENCES `adverbs`(`adverb_id`),
   FOREIGN KEY (`adverb_2_id`) REFERENCES `adverbs`(`adverb_id`)
 );
+
+DROP TABLE IF EXISTS `prepositions`;
+CREATE TABLE `prepositions` (
+  `preposition_id` INTEGER PRIMARY KEY,
+  `form` VARCHAR(32) NOT NULL
+);
+
+DROP TABLE IF EXISTS `preposition_groups`;
+CREATE TABLE `preposition_groups` (
+  `preposition_id` INTEGER NOT NULL,
+  `groupname` VARCHAR(32) NOT NULL,
+  FOREIGN KEY (`preposition_id`) REFERENCES `prepositions`(`preposition_id`)
+);
@@ -0,0 +1,482 @@
+diff /Users/hatkirby/Downloads/new_vn 2/admit-65.xml datadir/vn/admit-65.xml
+104c104
+<                     <SELRESTRS logic="or">
+---
+>                     <SELRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/amuse-31.1.xml datadir/vn/amuse-31.1.xml
+270a271,273
+>         <THEMROLE type="Cause">
+>           <SELRESTRS/>
+>         </THEMROLE>
+368c371,373
+<                 <LEX value="'s"/>
+---
+>                 <NP value="Cause">
+>                   <SYNRESTRS/>
+>                 </NP>
+404c409,411
+<                     <SYNRESTRS/>
+---
+>                     <SYNRESTRS>
+>                       <SYNRESTR Value="+" type="adjp"/>
+>                     </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/animal_sounds-38.xml datadir/vn/animal_sounds-38.xml
+186a187,191
+>                 <PREP>
+>                   <SELRESTRS>
+>                     <SELRESTR Value="+" type="loc" />
+>                   </SELRESTRS>
+>                 </PREP>
+diff /Users/hatkirby/Downloads/new_vn 2/assessment-34.1.xml datadir/vn/assessment-34.1.xml
+103d102
+<                 <LEX value="'s"/>
+diff /Users/hatkirby/Downloads/new_vn 2/battle-36.4.xml datadir/vn/battle-36.4.xml
+96c96
+<                         <SYNRESTR Value="+" type="what_extract"/>
+---
+>                         <SYNRESTR Value="+" type="wh_comp"/>
+diff /Users/hatkirby/Downloads/new_vn 2/become-109.1.xml datadir/vn/become-109.1.xml
+34c34,36
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/beg-58.2.xml datadir/vn/beg-58.2.xml
+41c41
+<                       <SYNRESTR Value="-" type="for_comp"/>
+---
+>                       <SYNRESTR Value="+" type="for_comp"/>
+diff /Users/hatkirby/Downloads/new_vn 2/bend-45.2.xml datadir/vn/bend-45.2.xml
+47c47,49
+<                 <SELRESTRS/>
+---
+>                 <SELRESTRS>
+>                   <SELRESTR Value="+" type="state"/>
+>                 </SELRESTRS>
+230c232,234
+<                     <SYNRESTRS/>
+---
+>                    <SYNRESTRS>
+>                      <SYNRESTR Value="+" type="adjp"/>
+>                    </SYNRESTRS>
+280,282c284
+<                     <SELRESTRS>
+<                         <SELRESTR Value="+" type="state"/>
+<                     </SELRESTRS>
+---
+>                     <SELRESTRS/>
+332,334c334
+<                     <SELRESTRS>
+<                         <SELRESTR Value="+" type="state"/>
+<                     </SELRESTRS>
+---
+>                     <SELRESTRS/>
+diff /Users/hatkirby/Downloads/new_vn 2/break-45.1.xml datadir/vn/break-45.1.xml
+255c255,257
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+307c309,311
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/characterize-29.2.xml datadir/vn/characterize-29.2.xml
+107c107
+<                 <LEX value="as"/>
+---
+>                 <LEX value="to be"/>
+109,111c109,111
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="+" type="small_clause"/>
+<                     </SYNRESTRS>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+386a387,391
+>                                 <NP value="Attribute">
+>                                   <SYNRESTRS>
+>                                     <SYNRESTR Value="+" type="adjp"/>
+>                                   </SYNRESTRS>
+>                                 </NP>
+diff /Users/hatkirby/Downloads/new_vn 2/coloring-24.xml datadir/vn/coloring-24.xml
+89c89,91
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/confess-37.10.xml datadir/vn/confess-37.10.xml
+110a111,115
+>                 <NP value="Attribute">
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+>                 </NP>
+diff /Users/hatkirby/Downloads/new_vn 2/consider-29.9.xml datadir/vn/consider-29.9.xml
+191,193c191,193
+<                                             <SYNRESTRS>
+<                                                 <SYNRESTR Value="-" type="sentential"/>
+<                                             </SYNRESTRS>
+---
+>                                   <SYNRESTRS>
+>                                     <SYNRESTR Value="+" type="adjp"/>
+>                                   </SYNRESTRS>
+334,336c334,336
+<                                             <SYNRESTRS>
+<                                                 <SYNRESTR Value="-" type="sentential"/>
+<                                             </SYNRESTRS>
+---
+>                                           <SYNRESTRS>
+>                                             <SYNRESTR Value="+" type="adjp"/>
+>                                           </SYNRESTRS>
+468,470c468,470
+<                                             <SYNRESTRS>
+<                                                 <SYNRESTR Value="-" type="sentential"/>
+<                                             </SYNRESTRS>
+---
+>                           <SYNRESTRS>
+>                             <SYNRESTR Value="+" type="adjp"/>
+>                           </SYNRESTRS>
+554,556c554,556
+<                                             <SYNRESTRS>
+<                                                 <SYNRESTR Value="-" type="sentential"/>
+<                                             </SYNRESTRS>
+---
+>                           <SYNRESTRS>
+>                             <SYNRESTR Value="+" type="adjp"/>
+>                           </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/cut-21.1.xml datadir/vn/cut-21.1.xml
+316c316,318
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+368c370,372
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+560c564,566
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/declare-29.4.xml datadir/vn/declare-29.4.xml
+33,35c33,35
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="-" type="sentential"/>
+<                     </SYNRESTRS>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+122,124c122,124
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="-" type="sentential"/>
+<                     </SYNRESTRS>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+244,246c244,246
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="-" type="sentential"/>
+<                     </SYNRESTRS>
+---
+>                           <SYNRESTRS>
+>                             <SYNRESTR Value="+" type="adjp"/>
+>                           </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/estimate-34.2.xml datadir/vn/estimate-34.2.xml
+123a124
+>                 <LEX value="to be"/>
+125,127c126,128
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="+" type="to_be"/>
+<                     </SYNRESTRS>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/get-13.5.1.xml datadir/vn/get-13.5.1.xml
+55,56c55
+<                 <SELRESTR Value="-" type="location"/>
+<                 <SELRESTR Value="-" type="region"/>
+---
+>                 <SELRESTR Value="+" type="currency"/>
+diff /Users/hatkirby/Downloads/new_vn 2/hit-18.1.xml datadir/vn/hit-18.1.xml
+234c234,236
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+294c296,298
+<                     <SYNRESTRS/>
+---
+>                    <SYNRESTRS>
+>                      <SYNRESTR Value="+" type="adjp"/>
+>                    </SYNRESTRS>
+619c623,625
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/instr_communication-37.4.xml datadir/vn/instr_communication-37.4.xml
+195c195,197
+<                     <SYNRESTRS/>
+---
+>                     <SYNRESTRS>
+>                       <SYNRESTR Value="+" type="quotation"/>
+>                     </SYNRESTRS>
+233c235,237
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                       <SYNRESTR Value="+" type="quotation"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/judgment-33.xml datadir/vn/judgment-33.xml
+187a188,190
+>                 <NP value="Attribute">
+>                   <SYNRESTRS/>
+>                 </NP>
+243a247
+>                 <LEX value="to be"/>
+245,247c249
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="+" type="small_clause"/>
+<                     </SYNRESTRS>
+---
+>                     <SYNRESTRS/>
+diff /Users/hatkirby/Downloads/new_vn 2/manner_speaking-37.3.xml datadir/vn/manner_speaking-37.3.xml
+264c264,266
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                       <SYNRESTR Value="+" type="quotation"/>
+>                   </SYNRESTRS>
+603c605,607
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                       <SYNRESTR Value="+" type="quotation"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/other_cos-45.4.xml datadir/vn/other_cos-45.4.xml
+534c534,536
+<                       <SYNRESTRS/>
+---
+>           <SYNRESTRS>
+>             <SYNRESTR Value="+" type="adjp"/>
+>           </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/pocket-9.10.xml datadir/vn/pocket-9.10.xml
+256c256,258
+<                             <SYNRESTRS/>
+---
+>                             <SYNRESTRS>
+>                               <SYNRESTR Value="+" type="adv-loc"/>
+>                             </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/poison-42.2.xml datadir/vn/poison-42.2.xml
+93c93,95
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/pour-9.5.xml datadir/vn/pour-9.5.xml
+59,61c59,62
+<                     <SELRESTRS>
+<                         <SELRESTR Value="+" type="path"/>
+<                         <SELRESTR Value="-" type="dest_dir"/>
+---
+>                     <SELRESTRS logic="or">
+>                         <SELRESTR Value="+" type="dir"/>
+>                         <SELRESTR Value="+" type="src"/>
+>                         <SELRESTR Value="+" type="dest_conf"/>
+157,160c158,162
+<                     <SELRESTRS>
+<                         <SELRESTR Value="+" type="path"/>
+<                         <SELRESTR Value="-" type="dest_dir"/>
+<                     </SELRESTRS>
+---
+>                   <SELRESTRS logic="or">
+>                       <SELRESTR Value="+" type="dir"/>
+>                       <SELRESTR Value="+" type="src"/>
+>                       <SELRESTR Value="+" type="dest_conf"/>
+>                   </SELRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/push-12.xml datadir/vn/push-12.xml
+90c90,92
+<                             <SYNRESTRS/>
+---
+>                           <SYNRESTRS>
+>                             <SYNRESTR Value="+" type="adjp"/>
+>                           </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/roll-51.3.1.xml datadir/vn/roll-51.3.1.xml
+190c190,192
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+256c258,260
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/see-30.1.xml datadir/vn/see-30.1.xml
+16a17,19
+>         <THEMROLE type="Patient">
+>           <SELRESTRS/>
+>         </THEMROLE>
+93a97,102
+>                 <PREP value="in">
+>                   <SELRESTRS/>
+>                 </PREP>
+>                 <NP value="Patient">
+>                   <SYNRESTRS/>
+>                 </NP>
+231a241,243
+>                         <NP value="Patient">
+>                           <SYNRESTRS/>
+>                         </NP>
+diff /Users/hatkirby/Downloads/new_vn 2/seem-109.xml datadir/vn/seem-109.xml
+30,32c30,32
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="-" type="sentential"/>
+<                     </SYNRESTRS>
+---
+>                    <SYNRESTRS>
+>                      <SYNRESTR Value="+" type="adjp"/>
+>                    </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/slide-11.2.xml datadir/vn/slide-11.2.xml
+69,72c69,73
+<                     <SELRESTRS>
+<                         <SELRESTR Value="+" type="path"/>
+<                         <SELRESTR Value="-" type="dest_dir"/>
+<                     </SELRESTRS>
+---
+>                   <SELRESTRS logic="or">
+>                       <SELRESTR Value="+" type="dir"/>
+>                       <SELRESTR Value="+" type="src"/>
+>                       <SELRESTR Value="+" type="dest_conf"/>
+>                   </SELRESTRS>
+218,221c219,223
+<                     <SELRESTRS>
+<                         <SELRESTR Value="+" type="path"/>
+<                         <SELRESTR Value="-" type="dest_dir"/>
+<                     </SELRESTRS>
+---
+>                   <SELRESTRS logic="or">
+>                       <SELRESTR Value="+" type="dir"/>
+>                       <SELRESTR Value="+" type="src"/>
+>                       <SELRESTR Value="+" type="dest_conf"/>
+>                   </SELRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/spank-18.3.xml datadir/vn/spank-18.3.xml
+69a70,72
+>         <THEMROLE type="Recipient">
+>           <SELRESTRS/>
+>         </THEMROLE>
+201c204,206
+<                     <SYNRESTRS/>
+---
+>                    <SYNRESTRS>
+>                      <SYNRESTR Value="+" type="adjp"/>
+>                    </SYNRESTRS>
+529,532c534,541
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="+" type="genitive"/>
+<                         <SYNRESTR Value="+" type="body_part"/>
+<                     </SYNRESTRS>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="genitive"/>
+>                   </SYNRESTRS>
+>                 </NP>
+>                 <NP value="Recipient">
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="body_part"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/swat-18.2.xml datadir/vn/swat-18.2.xml
+264c264,266
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+324c326,328
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/tape-22.4.xml datadir/vn/tape-22.4.xml
+364c364,366
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/vehicle-51.4.1.xml datadir/vn/vehicle-51.4.1.xml
+227c227,229
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/waltz-51.5.xml datadir/vn/waltz-51.5.xml
+181c181,183
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/want-32.1.xml datadir/vn/want-32.1.xml
+142a143
+>                         <ADV/>
+194a196
+>                         <ADJ/>
+305a308
+>                                 <LEX value="to be"/>
+307,309c310,312
+<                                     <SYNRESTRS>
+<                                         <SYNRESTR Value="+" type="to_be"/>
+<                                     </SYNRESTRS>
+---
+>                                   <SYNRESTRS>
+>                                     <SYNRESTR Value="+" type="adjp"/>
+>                                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/wipe_instr-10.4.2.xml datadir/vn/wipe_instr-10.4.2.xml
+178c178,180
+<                     <SYNRESTRS/>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+diff /Users/hatkirby/Downloads/new_vn 2/wipe_manner-10.4.1.xml datadir/vn/wipe_manner-10.4.1.xml
+198c198,199
+<                         <SELRESTR Value="-" type="region"/>
+---
+>                       <SELRESTR Value="+" type="location"/>
+>                       <SELRESTR Value="-" type="region"/>
+diff /Users/hatkirby/Downloads/new_vn 2/wish-62.xml datadir/vn/wish-62.xml
+91a92
+>                 <LEX value="to be"/>
+93,95c94,96
+<                     <SYNRESTRS>
+<                         <SYNRESTR Value="+" type="to_be"/>
+<                     </SYNRESTRS>
+---
+>                   <SYNRESTRS>
+>                     <SYNRESTR Value="+" type="adjp"/>
+>                   </SYNRESTRS>
+122a124
+>                 <ADJ/>