1 files changed, 73 insertions, 11 deletions

@@ -7,7 +7,71 @@
 #ifndef KGRAMSTATS_H
 #define KGRAMSTATS_H
 
-typedef std::list<std::string> kgram;
+struct token {
+  std::string canon;
+  bool terminating;
+  
+  token(std::string canon) : canon(canon), terminating(false) {}
+  
+  bool operator<(const token& other) const
+  {
+    if (canon == other.canon)
+    {
+      return !terminating && other.terminating;
+    } else {
+      return canon < other.canon;
+    }
+  }
+};
+
+enum querytype {
+  querytype_literal,
+  querytype_sentence
+};
+
+struct query {
+  querytype type;
+  token word;
+  
+  query(token word) : word(word), type(querytype_literal) {}
+  
+  query(querytype type) : word(""), type(type) {}
+  
+  bool operator<(const query& other) const
+  {
+    if (type == other.type)
+    {
+      return word < other.word;
+    } else {
+      return type < other.type;
+    }
+  }
+};
+
+typedef std::list<query> kgram;
+
+struct termstats {
+  char terminator;
+  int occurrences;
+  
+  termstats() : terminator('.'), occurrences(1) {}
+  
+  termstats(char terminator, int occurrences)
+  {
+    this->terminator = terminator;
+    this->occurrences = occurrences;
+  }
+  
+  bool operator<(const termstats& other) const
+  {
+    if (terminator == other.terminator)
+    {
+      return occurrences < other.occurrences;
+    } else {
+      return terminator < other.terminator;
+    }
+  }
+};
 
 class kgramstats
 {
@@ -16,22 +80,20 @@ public:
         std::vector<std::string> randomSentence(int n);
         
 private:
-        typedef struct
+        struct token_data
         {
                 int all;
                 int titlecase;
                 int uppercase;
-                int period;
-    int startquote;
-    int endquote;
-    int startparen;
-    int endparen;
-    int comma;
-                std::string* token;
-        } token_data;
+    token word;
+    
+    token_data() : word(""), all(0), titlecase(0), uppercase(0) {}
+        };
+  
         int maxK;
-        std::map<kgram, std::map<int, token_data*>* >* stats;
+        std::map<kgram, std::map<int, token_data> > stats;
   malaprop mstats;
+  std::map<token, std::map<int, termstats> > endings;
 };
 
 void printKgram(kgram k);