Rewrote how tokens are handled

A 'word' is now an object that contains a distribution of forms that word can take. For now, most word just contain one form, the canonical one. The only special use is currently hashtags.

Malapropisms have been disabled because of compatibility issues and because an upcoming feature is planned to replace it.

8 files changed, 406 insertions, 266 deletions

@@ -8,10 +8,14 @@ find_package(curl)
 if (YamlCpp_FOUND AND CURL_FOUND)
   add_subdirectory(vendor/twitcurl/libtwitcurl)
   include_directories(vendor/twitcurl/libtwitcurl)
-  add_executable(rawr-ebooks ebooks.cpp kgramstats.cpp freevars.cpp malaprop.cpp)
+  add_executable(rawr-ebooks ebooks.cpp kgramstats.cpp freevars.cpp histogram.cpp)
+  set_property(TARGET rawr-ebooks PROPERTY CXX_STANDARD 11)
+  set_property(TARGET rawr-ebooks PROPERTY CXX_STANDARD_REQUIRED ON)
   target_link_libraries(rawr-ebooks ${YamlCpp_LIBRARIES} twitcurl ${CURL_LIBRARIES})
 else (YamlCpp_FOUND AND CURL_FOUND)
   message(STATUS "rawr-ebooks requires yaml-cpp and twitcurl; without these, we will only make rawr-gen")
 endif (YamlCpp_FOUND AND CURL_FOUND)
 
-add_executable(rawr-gen gen.cpp kgramstats.cpp freevars.cpp malaprop.cpp)
+add_executable(rawr-gen gen.cpp kgramstats.cpp freevars.cpp histogram.cpp)
+set_property(TARGET rawr-gen PROPERTY CXX_STANDARD 11)
+set_property(TARGET rawr-gen PROPERTY CXX_STANDARD_REQUIRED ON)
@@ -44,20 +44,9 @@ int main(int argc, char** args)
   std::cout << "Generating..." << std::endl;
   for (;;)
   {
-    std::vector<std::string> doc = stats->randomSentence(rand() % 45 + 5);
-    std::string hi;
-    for (std::vector<std::string>::iterator it = doc.begin(); it != doc.end(); ++it)
-    {
-      hi += vars->parse(*it) + " ";
-    }
-
+    std::string doc = stats->randomSentence(rand() % 45 + 5);
+    std::string hi = vars->parse(doc);
     hi.resize(140);
-
-    size_t lastperiod = hi.find_last_of(".!?,");
-    if ((lastperiod != std::string::npos) && (rand() % 3 > 0))
-    {
-      hi = hi.substr(0, lastperiod+1);
-    }
     
     std::string replyMsg;
     if (twitter.statusUpdate(hi))
@@ -34,8 +34,8 @@ std::string freevars::parse(std::string in)
     for (std::map<std::string, std::vector<std::string>* >::iterator it = vars->begin(); it != vars->end(); it++)
     {
         std::string tofind = "$" + it->first + "$";
-        size_t fpos = res.find(tofind);
-        if (fpos != std::string::npos)
+        size_t fpos;
+        while ((fpos = res.find(tofind)) != std::string::npos)
         {
             int r = rand() % it->second->size();
             res.replace(fpos, tofind.length(), (*it->second)[r], 0, std::string::npos);
@@ -52,21 +52,10 @@ int main(int argc, char** args)
   std::cout << "Generating..." << std::endl;
   for (;;)
   {
-    std::vector<std::string> doc = stats->randomSentence(rand() % 35 + 15);
-    std::string hi;
-    for (std::vector<std::string>::iterator it = doc.begin(); it != doc.end(); ++it)
-    {
-      hi += vars->parse(*it) + " ";
-    }
-    
+    std::string doc = stats->randomSentence(rand() % 35 + 15);
+    std::string hi = vars->parse(doc);
     hi.resize(140);
 
-    size_t lastperiod = hi.find_last_of(".!?,");
-    if ((lastperiod != std::string::npos) && (rand() % 3 > 0))
-    {
-      hi = hi.substr(0, lastperiod+1);
-    }
-
     std::cout << hi << std::endl;
                 
     getc(stdin);
@@ -0,0 +1,34 @@
+#include "histogram.h"
+#include <cstdlib>
+
+template <class T>
+void histogram<T>::add(const T& inst)
+{
+  freqtable[inst]++;
+}
+
+template <class T>
+void histogram<T>::compile()
+{
+  distribution.clear();
+  
+  int max = 0;
+  for (auto& it : freqtable)
+  {
+    max += it.second;
+    distribution.emplace(max, it.first);
+  }
+  
+  freqtable.clear();
+}
+
+template <class T>
+const T& histogram<T>::next() const
+{
+  int max = distribution.rbegin()->first;
+  int r = rand() % max;
+  
+  return distribution.upper_bound(r)->second;
+}
+
+template class histogram <std::string>;
@@ -0,0 +1,19 @@
+#ifndef HISTOGRAM_H_24094D97
+#define HISTOGRAM_H_24094D97
+
+#include <map>
+#include <string>
+
+template <class T>
+class histogram {
+  public:
+    void add(const T& inst);
+    void compile();
+    const T& next() const;
+    
+  private:
+    std::map<T, int> freqtable;
+    std::map<int, T> distribution;
+};
+
+#endif /* end of include guard: HISTOGRAM_H_24094D97 */
@@ -37,35 +37,11 @@
 #include <iostream>
 #include <cstdlib>
 #include <algorithm>
-#include "malaprop.h"
+#include <set>
+#include <stack>
 
-query wildcardQuery(querytype_sentence);
-
-std::string canonize(std::string f);
-
-token token_from_string(std::string in)
-{
-  if (in[0] == '#')
-  {
-    token word(tokentype_hashtag);
-    
-    if (in.find_first_of(".?!,") != std::string::npos)
-    {
-      word.terminating = true;
-    }
-    
-    return word;
-  } else {
-    token word(canonize(in));
-  
-    if (in.find_first_of(".?!,") != std::string::npos)
-    {
-      word.terminating = true;
-    }
-    
-    return word;
-  }
-}
+query wildcardQuery {querytype::sentence};
+word blank_word {""};
 
 // runs in O(t^2) time where t is the number of tokens in the input corpus
 // We consider maxK to be fairly constant
@@ -73,7 +49,7 @@ kgramstats::kgramstats(std::string corpus, int maxK)
 {
   this->maxK = maxK;
   
-  std::vector<std::string> tokens;
+  std::vector<token> tokens;
   size_t start = 0;
   int end = 0;
   std::set<std::string> thashtags;
@@ -82,88 +58,186 @@ kgramstats::kgramstats(std::string corpus, int maxK)
   {
     end = corpus.find(" ", start);
 
-    std::string token = corpus.substr(start, (end == std::string::npos) ? std::string::npos : end - start);
-    if (token[token.length()-1] == '\n')
+    std::string t = corpus.substr(start, (end == std::string::npos) ? std::string::npos : end - start);
+    if (t.compare("") && t.compare("."))
     {
-      if ((token[token.length()-2] != '.') && (token[token.length()-2] != '!') && (token[token.length()-2] != '?') && (token[token.length()-2] != ','))
+      std::string tc(t), canonical;
+      std::transform(tc.begin(), tc.end(), tc.begin(), ::tolower);
+      std::remove_copy_if(tc.begin(), tc.end(), std::back_inserter(canonical), [=] (char c) {
+        return !((c != '.') && (c != '?') && (c != '!') && (c != ',') && (c != '"') && (c != '(') && (c != ')') && (c != '\n') && (c != '[') && (c != ']') && (c != '*'));
+      });
+      
+      word& w = ([&] () -> word& {
+        // Hashtag freevar
+        if (canonical[0] == '#')
+        {
+          thashtags.insert(canonical);
+          canonical = "#hashtag";
+          
+          return hashtags;
+        }
+        
+        // Basically any other word
+        if (words.count(canonical) == 0)
+        {
+          words.emplace(canonical, canonical);
+        }
+        
+        word& tw = words.at(canonical);
+        tw.forms.add(canonical);
+        
+        return tw;
+      })();
+      
+      token tk(w);
+      tk.raw = t;
+      
+      for (char c : t)
       {
-        token.insert(token.length()-1, ".");
+        if (c == '*')
+        {
+          tk.delimiters[{parentype::asterisk, doublestatus::opening}]++;
+        } else if (c == '[')
+        {
+          tk.delimiters[{parentype::square_bracket, doublestatus::opening}]++;
+        } else if (c == '(')
+        {
+          tk.delimiters[{parentype::paren, doublestatus::opening}]++;
+        } else if (c == '"')
+        {
+          tk.delimiters[{parentype::quote, doublestatus::opening}]++;
+        } else {
+          break;
+        }
       }
-         
-      token.resize(token.length()-1);
-    }
-       
-    if (token.compare("") && token.compare("."))
-    {
-      mstats.addWord(token);
-      tokens.push_back(token);
       
-      if (token[0] == '#')
+      int backtrack = t.find_last_not_of(".,?!])\"*\n") + 1;
+      if (backtrack != t.length())
       {
-        thashtags.insert(canonize(token));
+        std::string ending = t.substr(backtrack);
+        std::string suffix;
+        
+        for (char c : ending)
+        {
+          if ((c == '.') || (c == ',') || (c == '?') || (c == '!'))
+          {
+            suffix += c;
+            
+            continue;
+          } else if (c == '\n')
+          {
+            // At least the end is coming
+            if (suffix.empty())
+            {
+              suffix = ".";
+            }
+            
+            break;
+          }
+          
+          parentype pt = ([&] {
+            switch (c)
+            {
+              case ']': return parentype::square_bracket;
+              case ')': return parentype::paren;
+              case '*': return parentype::asterisk;
+              case '"': return parentype::quote;
+            }
+          })();
+          
+          if (tk.delimiters[{pt, doublestatus::opening}] > 0)
+          {
+            tk.delimiters[{pt, doublestatus::opening}]--;
+            tk.delimiters[{pt, doublestatus::both}]++;
+          } else {
+            tk.delimiters[{pt, doublestatus::closing}]++;
+          }
+        }
+        
+        if (suffix == ",")
+        {
+          tk.suffix = suffixtype::comma;
+        } else if (!suffix.empty()) {
+          tk.suffix = suffixtype::terminating;
+          
+          w.terms.add(suffix);
+        }
       }
+      
+      tokens.push_back(tk);
     }
 
     start = ((end > (std::string::npos - 1) ) ? std::string::npos : end + 1);
   }
   
-  for (std::set<std::string>::iterator it = thashtags.begin(); it != thashtags.end(); it++)
+  // Time to condense the distribution stuff for the words
+  for (auto& it : words)
   {
-    hashtags.push_back(*it);
+    it.second.forms.compile();
+    it.second.terms.compile();
   }
-        
+  
+  // Hashtag freevar is not frequency distributed
+  for (auto& it : thashtags)
+  {
+    hashtags.forms.add(it);
+  }
+  
+  hashtags.forms.compile();
+  hashtags.terms.compile();
+
+  // kgram distribution
   std::map<kgram, std::map<token, token_data> > tstats;
-  std::map<token, std::map<termstats, int> > tendings;
   for (int k=1; k<maxK; k++)
   {
     for (int i=0; i<(tokens.size() - k); i++)
     {
-      std::list<std::string> seq(tokens.begin()+i, tokens.begin()+i+k);
-      kgram prefix;
-      
-      for (std::list<std::string>::iterator it = seq.begin(); it != seq.end(); it++)
-      {
-        prefix.push_back(token_from_string(*it));
-      }
-      
-      std::string f = tokens[i+k];
-                  std::string canonical = canonize(f);
-      
-      token word(token_from_string(canonical));
-      if (f.find_first_of(".?!,") != std::string::npos)
+      kgram prefix(tokens.begin()+i, tokens.begin()+i+k);
+      token f = tokens[i+k];
+
+      if (tstats[prefix].count(f) == 0)
       {
-        word.terminating = true;
-        
-        char terminator = f[f.find_last_of(".?!,")];
-        int occurrences = std::count(f.begin(), f.end(), terminator);
-        
-        tendings[word][termstats(terminator, occurrences)]++;
+        tstats[prefix].emplace(f, f);
       }
                         
-      token_data& td = tstats[prefix][word];
-      td.word = word;
+      token_data& td = tstats[prefix].at(f);
       td.all++;
 
-      if (std::find_if(f.begin(), f.end(), ::islower) == f.end())
+      if (std::find_if(f.raw.begin(), f.raw.end(), ::islower) == f.raw.end())
       {
         td.uppercase++;
-      } else if (isupper(f[0]))
+      } else if (isupper(f.raw[0]))
       {
         td.titlecase++;
       }
       
-      if (prefix.front().word.terminating)
+      kgram term_prefix;
+      bool changed = false;
+      std::transform(prefix.begin(), prefix.end(), std::back_inserter(term_prefix), [&] (query& q) {
+        if (q.tok.suffix == suffixtype::terminating)
+        {
+          changed = true;
+          
+          return wildcardQuery;
+        } else {
+          return q;
+        }
+      });
+      
+      if (changed)
       {
-        prefix.front() = wildcardQuery;
+        if (tstats[term_prefix].count(f) == 0)
+        {
+          tstats[term_prefix].emplace(f, f);
+        }
         
-        token_data& td2 = tstats[prefix][word];
-        td2.word = word;
+        token_data& td2 = tstats[term_prefix].at(f);
         td2.all++;
 
-        if (std::find_if(f.begin(), f.end(), ::islower) == f.end())
+        if (std::find_if(f.raw.begin(), f.raw.end(), ::islower) == f.raw.end())
         {
           td2.uppercase++;
-        } else if (isupper(f[0]))
+        } else if (isupper(f.raw[0]))
         {
           td2.titlecase++;
         }
@@ -171,74 +245,52 @@ kgramstats::kgramstats(std::string corpus, int maxK)
     }
   }
         
-  for (std::map<kgram, std::map<token, token_data> >::iterator it = tstats.begin(); it != tstats.end(); it++)
+  // Condense the kgram distribution
+  for (auto& it : tstats)
   {
-    kgram klist = it->first;
-    std::map<token, token_data>& probtable = it->second;
-    std::map<int, token_data>& distribution = stats[klist];
+    kgram klist = it.first;
+    auto& probtable = it.second;
+    auto& distribution = stats[klist];
     int max = 0;
                 
-    for (std::map<token, token_data>::iterator kt = probtable.begin(); kt != probtable.end(); kt++)
+    for (auto& kt : probtable)
     {
-      max += kt->second.all;
+      max += kt.second.all;
                         
-      distribution[max] = kt->second;
-    }
-  }
-  
-  for (std::map<token, std::map<termstats, int> >::iterator it = tendings.begin(); it != tendings.end(); it++)
-  {
-    token word = it->first;
-    std::map<termstats, int>& probtable = it->second;
-    std::map<int, termstats>& distribution = endings[word];
-    int max = 0;
-    
-    for (std::map<termstats, int>::iterator kt = probtable.begin(); kt != probtable.end(); kt++)
-    {
-      max += kt->second;
-      
-      distribution[max] = kt->first;
+      distribution.emplace(max, kt.second);
     }
   }
 }
 
 void printKgram(kgram k)
 {
-  for (kgram::iterator it = k.begin(); it != k.end(); it++)
+  for (auto& q : k)
   {
-    query& q = *it;
-    if (q.type == querytype_sentence)
+    if (q.type == querytype::sentence)
     {
       std::cout << "#.# ";
-    } else if (q.type == querytype_literal)
+    } else if (q.type == querytype::literal)
     {
-      if (q.word.type == tokentype_hashtag)
+      if (q.tok.suffix == suffixtype::terminating)
       {
-        if (q.word.terminating)
-        {
-          std::cout << "#hashtag. ";
-        } else {
-          std::cout << "#hashtag ";
-        }
-      } else if (q.word.type == tokentype_literal)
+        std::cout << q.tok.w.canon << ". ";
+      } else if (q.tok.suffix == suffixtype::comma)
       {
-        if (q.word.terminating)
-        {
-          std::cout << q.word.canon << ". ";
-        } else {
-          std::cout << q.word.canon << " ";
-        }
+        std::cout << q.tok.w.canon << ", ";
+      } else {
+        std::cout << q.tok.w.canon << " ";
       }
     }
   }
 }
 
 // runs in O(n log t) time where n is the input number of sentences and t is the number of tokens in the input corpus
-std::vector<std::string> kgramstats::randomSentence(int n)
+std::string kgramstats::randomSentence(int n)
 {
-  std::vector<std::string> result;
+  std::string result;
   kgram cur(1, wildcardQuery);
   int cuts = 0;
+  std::stack<parentype> open_delimiters;
         
   for (int i=0; i<n; i++)
   {
@@ -273,86 +325,135 @@ std::vector<std::string> kgramstats::randomSentence(int n)
       cur = kgram(1, wildcardQuery);
     }
 
-    std::map<int, token_data>& distribution = stats[cur];
+    auto& distribution = stats[cur];
     int max = distribution.rbegin()->first;
     int r = rand() % max;
     token_data& next = distribution.upper_bound(r)->second;
-    std::string nextToken;
-    bool mess = false;
-    
-    if (next.word.type == tokentype_literal)
+    std::string nextToken = next.tok.w.forms.next();
+  
+    // Determine the casing of the next token. We randomly make the token all
+    // caps based on the markov chain. Otherwise, we check if the previous
+    // token is the end of a sentence (terminating token or a wildcard query).
+    int casing = rand() % next.all;
+    if (casing < next.uppercase)
+    {
+      std::transform(nextToken.begin(), nextToken.end(), nextToken.begin(), ::toupper);
+    } else if ((((cur.rbegin()->type == querytype::sentence)
+          || ((cur.rbegin()->type == querytype::literal)
+            && (cur.rbegin()->tok.suffix == suffixtype::terminating)))
+        && (rand() % 2 > 0))
+      || (casing - next.uppercase < next.titlecase))
     {
-      nextToken = next.word.canon;
+      nextToken[0] = toupper(nextToken[0]);
+    }
     
-      mess = (rand() % 100) == 0;
-      if (mess)
+    // Delimiters
+    for (auto& dt : next.tok.delimiters)
+    {
+      if (dt.first.status == doublestatus::both)
       {
-        nextToken = mstats.alternate(nextToken);
-      }
-    
-      // Determine the casing of the next token. We randomly make the token all
-      // caps based on the markov chain. Otherwise, we check if the previous
-      // token is the end of a sentence (terminating token or a wildcard query).
-      int casing = rand() % next.all;
-      if (casing < next.uppercase)
+        switch (dt.first.type)
+        {
+          case parentype::paren: nextToken = std::string("(", dt.second) + nextToken + std::string(")", dt.second); break;
+          case parentype::square_bracket: nextToken = std::string("[", dt.second) + nextToken + std::string("]", dt.second); break;
+          case parentype::asterisk: nextToken = std::string("*", dt.second) + nextToken + std::string("*", dt.second); break;
+          case parentype::quote: nextToken = std::string("\"", dt.second) + nextToken + std::string("\"", dt.second); break;
+        }
+      } else if (dt.first.status == doublestatus::opening)
       {
-        std::transform(nextToken.begin(), nextToken.end(), nextToken.begin(), ::toupper);
-      } else if ((((cur.rbegin()->type == querytype_sentence)
-            || ((cur.rbegin()->type == querytype_literal)
-              && (cur.rbegin()->word.terminating)))
-          && (rand() % 2 > 0))
-        || (casing - next.uppercase < next.titlecase))
+        for (int i=0; i<dt.second; i++)
+        {
+          open_delimiters.push(dt.first.type);
+        }
+        
+        switch (dt.first.type)
+        {
+          case parentype::paren: nextToken = std::string("(", dt.second) + nextToken; break;
+          case parentype::square_bracket: nextToken = std::string("[", dt.second) + nextToken; break;
+          case parentype::asterisk: nextToken = std::string("*", dt.second) + nextToken; break;
+          case parentype::quote: nextToken = std::string("\"", dt.second) + nextToken; break;
+        }
+      } else if (dt.first.status == doublestatus::closing)
       {
-        nextToken[0] = toupper(nextToken[0]);
+        for (int i=0; i<dt.second; i++)
+        {
+          while (!open_delimiters.empty() && (open_delimiters.top() != dt.first.type))
+          {
+            switch (open_delimiters.top())
+            {
+              case parentype::paren: nextToken.append(")"); break;
+              case parentype::square_bracket: nextToken.append("]"); break;
+              case parentype::asterisk: nextToken.append("*"); break;
+              case parentype::quote: nextToken.append("\""); break;
+            }
+            
+            open_delimiters.pop();
+          }
+          
+          if (open_delimiters.empty())
+          {
+            switch (dt.first.type)
+            {
+              case parentype::paren: result = "(" + result; break;
+              case parentype::square_bracket: result = "[" + result; break;
+              case parentype::asterisk: result = "*" + result; break;
+              case parentype::quote: result = "\"" + result; break;
+            }
+          }
+          
+          switch (dt.first.type)
+          {
+            case parentype::paren: nextToken.append(")"); break;
+            case parentype::square_bracket: nextToken.append("]"); break;
+            case parentype::asterisk: nextToken.append("*"); break;
+            case parentype::quote: nextToken.append("\""); break;
+          }
+        }
       }
-    } else if (next.word.type == tokentype_hashtag)
-    {
-      int rhash = rand() % hashtags.size();
-      nextToken = hashtags[rhash];
     }
     
-    if (next.word.terminating)
+    // Terminators
+    if (next.tok.suffix == suffixtype::terminating)
     {
-      std::map<int, termstats>& ending = endings[next.word];
-      int emax = ending.rbegin()->first;
-      int er = rand() % emax;
-      termstats& nextend = ending.upper_bound(er)->second;
-    
-      nextToken.append(std::string(nextend.occurrences, nextend.terminator));
+      nextToken.append(next.tok.w.terms.next());
+    } else if (next.tok.suffix == suffixtype::comma)
+    {
+      nextToken.append(",");
     }
                 
     /* DEBUG */
     printKgram(cur);
+    std::cout << "-> \"" << nextToken << "\" (" << next.all << "/" << max << ")" << std::endl;
+
+    cur.push_back(next.tok);
                 
-    std::cout << "-> \"" << nextToken << "\" (" << next.all << "/" << max << ")";
+    result.append(nextToken + " ");
     
-    if (mess)
+    if ((next.tok.suffix == suffixtype::terminating) && (rand() % 4 == 0))
     {
-      std::cout << " mala " << next.word.canon;
+      break;
     }
-    
-    std::cout << std::endl;
-
-    cur.push_back(next.word);
-                
-    result.push_back(nextToken);
   }
-        
-  return result;
-}
-
-bool removeIf(char c)
-{
-  return !((c != '.') && (c != '?') && (c != '!') && (c != ',') /*&& (c != '"') && (c != '(') && (c != ')') && (c != '\n')*/);
-}
-
-std::string canonize(std::string f)
-{
-  std::string canonical(f);
-  std::transform(canonical.begin(), canonical.end(), canonical.begin(), ::tolower);
   
-  std::string result;
-  std::remove_copy_if(canonical.begin(), canonical.end(), std::back_inserter(result), removeIf);
+  // Remove the trailing space
+  if (result.back() == ' ')
+  {
+    result.pop_back();
+  }
+  
+  // Close any open delimiters
+  while (!open_delimiters.empty())
+  {
+    switch (open_delimiters.top())
+    {
+      case parentype::paren: result.append(")"); break;
+      case parentype::square_bracket: result.append("]"); break;
+      case parentype::asterisk: result.append("*"); break;
+      case parentype::quote: result.append("\""); break;
+    }
+    
+    open_delimiters.pop();
+  }
         
   return result;
 }
@@ -2,61 +2,89 @@
 #include <map>
 #include <list>
 #include <vector>
-#include "malaprop.h"
+#include "histogram.h"
 
 #ifndef KGRAMSTATS_H
 #define KGRAMSTATS_H
 
-enum tokentype {
-  tokentype_literal,
-  tokentype_hashtag
+struct word {
+  std::string canon;
+  histogram<std::string> forms;
+  histogram<std::string> terms;
+  
+  word(std::string canon) : canon(canon) {}
+  
+  bool operator<(const word& other) const
+  {
+    return canon < other.canon;
+  }
 };
 
-struct token {
-  tokentype type;
-  std::string canon;
-  bool terminating;
+extern word blank_word;
+
+enum class suffixtype {
+  none,
+  terminating,
+  comma
+};
+
+enum class parentype {
+  paren,
+  square_bracket,
+  asterisk,
+  quote
+};
+
+enum class doublestatus {
+  opening,
+  closing,
+  both
+};
+
+struct delimiter {
+  parentype type;
+  doublestatus status;
+  
+  delimiter(parentype type, doublestatus status) : type(type), status(status) {}
   
-  token(std::string canon) : type(tokentype_literal), canon(canon), terminating(false) {}
-  token(tokentype type) : type(type), canon(""), terminating(false) {}
+  bool operator<(const delimiter& other) const
+  {
+    return std::tie(type, status) < std::tie(other.type, other.status);
+  }
+};
+
+struct token {
+  const word& w;
+  std::map<delimiter, int> delimiters;
+  suffixtype suffix;
+  std::string raw;
+    
+  token(const word& w) : w(w), suffix(suffixtype::none) {}
   
   bool operator<(const token& other) const
   {
-    if (type != other.type)
-    {
-      return type < other.type;
-    } else if (type == tokentype_literal)
-    {
-      if (canon == other.canon)
-      {
-        return !terminating && other.terminating;
-      } else {
-        return canon < other.canon;
-      }
-    } else {
-      return !terminating && other.terminating;
-    }
+    return std::tie(w, delimiters, suffix) < std::tie(other.w, other.delimiters, other.suffix);
   }
 };
 
-enum querytype {
-  querytype_literal,
-  querytype_sentence
+enum class querytype {
+  literal,
+  sentence
 };
 
 struct query {
   querytype type;
-  token word;
+  token tok;
   
-  query(token word) : word(word), type(querytype_literal) {}
+  query(token tok) : tok(tok), type(querytype::literal) {}
   
-  query(querytype type) : word(""), type(type) {}
+  query(querytype type) : tok(blank_word), type(type) {}
   
   bool operator<(const query& other) const
   {
     if (type == other.type)
     {
-      return word < other.word;
+      return tok < other.tok;
     } else {
       return type < other.type;
     }
@@ -65,34 +93,11 @@ struct query {
 
 typedef std::list<query> kgram;
 
-struct termstats {
-  char terminator;
-  int occurrences;
-  
-  termstats() : terminator('.'), occurrences(1) {}
-  
-  termstats(char terminator, int occurrences)
-  {
-    this->terminator = terminator;
-    this->occurrences = occurrences;
-  }
-  
-  bool operator<(const termstats& other) const
-  {
-    if (terminator == other.terminator)
-    {
-      return occurrences < other.occurrences;
-    } else {
-      return terminator < other.terminator;
-    }
-  }
-};
-
 class kgramstats
 {
 public:
         kgramstats(std::string corpus, int maxK);
-        std::vector<std::string> randomSentence(int n);
+        std::string randomSentence(int n);
         
 private:
         struct token_data
@@ -100,16 +105,15 @@ private:
                 int all;
                 int titlecase;
                 int uppercase;
-    token word;
+    token tok;
     
-    token_data() : word(""), all(0), titlecase(0), uppercase(0) {}
+    token_data(token tok) : tok(tok), all(0), titlecase(0), uppercase(0) {}
         };
   
         int maxK;
         std::map<kgram, std::map<int, token_data> > stats;
-  malaprop mstats;
-  std::map<token, std::map<int, termstats> > endings;
-  std::vector<std::string> hashtags;
+  word hashtags {"#hashtag"};
+  std::map<std::string, word> words;
 };
 
 void printKgram(kgram k);