Rewrote quite a bit of kgramstats

The algorithm still treats most tokens literally, but now groups together tokens that terminate a clause somehow (so, contain .?!,), without distinguishing between the different terminating characters. For each word that can terminate a sentence, the algorithm creates a histogram of the terminating characters and number of occurrences of those characters for that word (number of occurrences is to allow things like um???? and um,,,,, to still be folded down into um.). Then, when the terminating version of that token is invoked, a random terminating string is added to that token based on the histogram for that word (again, to allow things like the desu-ly use of multiple commas to end clauses).

The algorithm now also has a slightly advanced kgram structure; a special "sentence wildcard" kgram value is set aside from normal strings of tokens that can match any terminating token. This kgram value is never printed (it is only ever present in the query kgrams and cannot actually be present in the histograms (it is of a different datatype)) and is used at the beginning of sentence generation to make sure that the first couple of words generated actually form the beginning of a sentence instead of picking up somewhere in the middle of a sentence. It is also used to reset sentence generation in the rare occasion that the end of the corpus is reached.

1 files changed, 73 insertions, 11 deletions

@@ -7,7 +7,71 @@
 #ifndef KGRAMSTATS_H
 #define KGRAMSTATS_H
 
-typedef std::list<std::string> kgram;
+struct token {
+  std::string canon;
+  bool terminating;
+  
+  token(std::string canon) : canon(canon), terminating(false) {}
+  
+  bool operator<(const token& other) const
+  {
+    if (canon == other.canon)
+    {
+      return !terminating && other.terminating;
+    } else {
+      return canon < other.canon;
+    }
+  }
+};
+
+enum querytype {
+  querytype_literal,
+  querytype_sentence
+};
+
+struct query {
+  querytype type;
+  token word;
+  
+  query(token word) : word(word), type(querytype_literal) {}
+  
+  query(querytype type) : word(""), type(type) {}
+  
+  bool operator<(const query& other) const
+  {
+    if (type == other.type)
+    {
+      return word < other.word;
+    } else {
+      return type < other.type;
+    }
+  }
+};
+
+typedef std::list<query> kgram;
+
+struct termstats {
+  char terminator;
+  int occurrences;
+  
+  termstats() : terminator('.'), occurrences(1) {}
+  
+  termstats(char terminator, int occurrences)
+  {
+    this->terminator = terminator;
+    this->occurrences = occurrences;
+  }
+  
+  bool operator<(const termstats& other) const
+  {
+    if (terminator == other.terminator)
+    {
+      return occurrences < other.occurrences;
+    } else {
+      return terminator < other.terminator;
+    }
+  }
+};
 
 class kgramstats
 {
@@ -16,22 +80,20 @@ public:
         std::vector<std::string> randomSentence(int n);
         
 private:
-        typedef struct
+        struct token_data
         {
                 int all;
                 int titlecase;
                 int uppercase;
-                int period;
-    int startquote;
-    int endquote;
-    int startparen;
-    int endparen;
-    int comma;
-                std::string* token;
-        } token_data;
+    token word;
+    
+    token_data() : word(""), all(0), titlecase(0), uppercase(0) {}
+        };
+  
         int maxK;
-        std::map<kgram, std::map<int, token_data*>* >* stats;
+        std::map<kgram, std::map<int, token_data> > stats;
   malaprop mstats;
+  std::map<token, std::map<int, termstats> > endings;
 };
 
 void printKgram(kgram k);