Weighed token casing and presence of periods

Tokens which differ only by casing or the presence of an ending period are
now considered the same token. When tokens are generated, they are cased
based on the prevalence of Upper/Title/Lower casing of the token in the
input corpus, and similarly, a period is added to the end of the word based
on how often the same token was ended with a period in the input corpus.

2 files changed, 76 insertions, 28 deletions

@@ -2,6 +2,7 @@
 #include <vector>
 #include <iostream>
 #include <cstdlib>
+#include <algorithm>
 
 kgramstats::kgramstats(string corpus, int maxK)
 {
@@ -20,34 +21,45 @@ kgramstats::kgramstats(string corpus, int maxK)
            start = ((end > (string::npos - 1) ) ? string::npos : end + 1);
         }
         
-        stats = new map<kgram, map<string, int>* >();
+        stats = new map<kgram, map<string, token_data*>* >();
         for (int k=0; k<=maxK; k++)
         {
                 for (int i=0; i<(tokens.size() - k); i++)
                 {
                         kgram seq(tokens.begin()+i, tokens.begin()+i+k);
+                        transform(seq.begin(), seq.end(), seq.begin(), canonize);
                         string f = tokens[i+k];
+                        string canonical = canonize(f);
                         
                         if ((*stats)[seq] == NULL)
                         {
-                                (*stats)[seq] = new map<string, int>();
+                                (*stats)[seq] = new map<string, token_data*>();
                         }
                         
-                        (*((*stats)[seq]))[f]++;
+                        if ((*(*stats)[seq])[canonical] == NULL)
+                        {
+                                (*(*stats)[seq])[canonical] = (token_data*) calloc(1, sizeof(token_data));
+                        }
+
+                        token_data* td = stats->at(seq)->at(canonical);
+                        td->all++;
+                        
+                        if ((f.length() > 0) && (f[f.length()-1] == '.'))
+                        {
+                                td->period++;
+                        }
+                        
+                        if (std::find_if(f.begin(), f.end(), ::islower) == f.end())
+                        {
+                                td->uppercase++;
+                        } else if (isupper(f[0]))
+                        {
+                                td->titlecase++;
+                        }
                 }
         }
 }
 
-map<string, int>* kgramstats::lookupExts(kgram tk)
-{
-        return (*stats)[tk];
-}
-
-int kgramstats::getMaxK()
-{
-        return maxK;
-}
-
 void printKgram(kgram k)
 {
         for (kgram::iterator it = k.begin(); it != k.end(); it++)
@@ -76,35 +88,65 @@ vector<string> kgramstats::randomSentence(int n)
                                 }
                         }
                 }
-                
-                map<string, int>* probtable = lookupExts(cur);
+
+                map<string, token_data*>* probtable = (*stats)[cur];
                 int max = 0;
-                for (map<string, int>::iterator it = probtable->begin(); it != probtable->end(); ++it)
+                for (map<string, token_data*>::iterator it = probtable->begin(); it != probtable->end(); ++it)
                 {
-                        max += it->second;
+                        max += it->second->all;
                 }
-                
+
                 int r = rand() % (max+1);
-                string next = probtable->begin()->first;
-                for (map<string, int>::iterator it = probtable->begin(); it != probtable->end(); ++it)
+                map<string, token_data*>::iterator next = probtable->begin();
+                for (map<string, token_data*>::iterator it = probtable->begin(); it != probtable->end(); ++it)
                 {
-                        if (it->second > r)
+                        if (it->second->all > r)
                         {
                                 break;
                         } else {
-                                next = it->first;
-                                r -= it->second;
+                                next = it;
+                                r -= it->second->all;
                         }
                 }
 
+                string nextToken(next->first);
+                int casing = rand() % next->second->all;
+                int period = rand() % next->second->all;
+                if (casing < next->second->uppercase)
+                {
+                        transform(nextToken.begin(), nextToken.end(), nextToken.begin(), ::toupper);
+                } else if ((casing - next->second->uppercase) < next->second->titlecase)
+                {
+                        nextToken[0] = toupper(nextToken[0]);
+                }
+                
+                if (period < next->second->period)
+                {
+                        nextToken += ".";
+                }
+                
+                cout << next->first << " | " << nextToken << endl;
+
                 if (cur.size() == maxK)
                 {
                         cur.pop_front();
                 }
 
-                cur.push_back(next);
-                result.push_back(next);
+                cur.push_back(next->first);
+                result.push_back(nextToken);
         }
         
         return result;
+}
+
+std::string canonize(std::string f)
+{
+        string canonical(f);
+        transform(canonical.begin(), canonical.end(), canonical.begin(), ::tolower);
+        if (canonical[canonical.length()-1] == '.')
+        {
+                canonical.resize(canonical.find('.'));
+        }
+        
+        return canonical;
 }
\ No newline at end of file
@@ -14,15 +14,21 @@ class kgramstats
 {
 public:
         kgramstats(string corpus, int maxK);
-        map<string, int>* lookupExts(kgram tk);
-        int getMaxK();
         vector<string> randomSentence(int n);
         
 private:
+        typedef struct
+        {
+                int all;
+                int titlecase;
+                int uppercase;
+                int period;
+        } token_data;
         int maxK;
-        map<kgram, map<string, int>* >* stats;
+        map<kgram, map<string, token_data*>* >* stats;
 };
 
 void printKgram(kgram k);
+std::string canonize(std::string f);
 
 #endif
\ No newline at end of file