2 files changed, 76 insertions, 28 deletions

@@ -2,6 +2,7 @@
 #include <vector>
 #include <iostream>
 #include <cstdlib>
+#include <algorithm>
 
 kgramstats::kgramstats(string corpus, int maxK)
 {
@@ -20,34 +21,45 @@ kgramstats::kgramstats(string corpus, int maxK)
            start = ((end > (string::npos - 1) ) ? string::npos : end + 1);
         }
         
-        stats = new map<kgram, map<string, int>* >();
+        stats = new map<kgram, map<string, token_data*>* >();
         for (int k=0; k<=maxK; k++)
         {
                 for (int i=0; i<(tokens.size() - k); i++)
                 {
                         kgram seq(tokens.begin()+i, tokens.begin()+i+k);
+                        transform(seq.begin(), seq.end(), seq.begin(), canonize);
                         string f = tokens[i+k];
+                        string canonical = canonize(f);
                         
                         if ((*stats)[seq] == NULL)
                         {
-                                (*stats)[seq] = new map<string, int>();
+                                (*stats)[seq] = new map<string, token_data*>();
                         }
                         
-                        (*((*stats)[seq]))[f]++;
+                        if ((*(*stats)[seq])[canonical] == NULL)
+                        {
+                                (*(*stats)[seq])[canonical] = (token_data*) calloc(1, sizeof(token_data));
+                        }
+
+                        token_data* td = stats->at(seq)->at(canonical);
+                        td->all++;
+                        
+                        if ((f.length() > 0) && (f[f.length()-1] == '.'))
+                        {
+                                td->period++;
+                        }
+                        
+                        if (std::find_if(f.begin(), f.end(), ::islower) == f.end())
+                        {
+                                td->uppercase++;
+                        } else if (isupper(f[0]))
+                        {
+                                td->titlecase++;
+                        }
                 }
         }
 }
 
-map<string, int>* kgramstats::lookupExts(kgram tk)
-{
-        return (*stats)[tk];
-}
-
-int kgramstats::getMaxK()
-{
-        return maxK;
-}
-
 void printKgram(kgram k)
 {
         for (kgram::iterator it = k.begin(); it != k.end(); it++)
@@ -76,35 +88,65 @@ vector<string> kgramstats::randomSentence(int n)
                                 }
                         }
                 }
-                
-                map<string, int>* probtable = lookupExts(cur);
+
+                map<string, token_data*>* probtable = (*stats)[cur];
                 int max = 0;
-                for (map<string, int>::iterator it = probtable->begin(); it != probtable->end(); ++it)
+                for (map<string, token_data*>::iterator it = probtable->begin(); it != probtable->end(); ++it)
                 {
-                        max += it->second;
+                        max += it->second->all;
                 }
-                
+
                 int r = rand() % (max+1);
-                string next = probtable->begin()->first;
-                for (map<string, int>::iterator it = probtable->begin(); it != probtable->end(); ++it)
+                map<string, token_data*>::iterator next = probtable->begin();
+                for (map<string, token_data*>::iterator it = probtable->begin(); it != probtable->end(); ++it)
                 {
-                        if (it->second > r)
+                        if (it->second->all > r)
                         {
                                 break;
                         } else {
-                                next = it->first;
-                                r -= it->second;
+                                next = it;
+                                r -= it->second->all;
                         }
                 }
 
+                string nextToken(next->first);
+                int casing = rand() % next->second->all;
+                int period = rand() % next->second->all;
+                if (casing < next->second->uppercase)
+                {
+                        transform(nextToken.begin(), nextToken.end(), nextToken.begin(), ::toupper);
+                } else if ((casing - next->second->uppercase) < next->second->titlecase)
+                {
+                        nextToken[0] = toupper(nextToken[0]);
+                }
+                
+                if (period < next->second->period)
+                {
+                        nextToken += ".";
+                }
+                
+                cout << next->first << " | " << nextToken << endl;
+
                 if (cur.size() == maxK)
                 {
                         cur.pop_front();
                 }
 
-                cur.push_back(next);
-                result.push_back(next);
+                cur.push_back(next->first);
+                result.push_back(nextToken);
         }
         
         return result;
+}
+
+std::string canonize(std::string f)
+{
+        string canonical(f);
+        transform(canonical.begin(), canonical.end(), canonical.begin(), ::tolower);
+        if (canonical[canonical.length()-1] == '.')
+        {
+                canonical.resize(canonical.find('.'));
+        }
+        
+        return canonical;
 }
\ No newline at end of file
@@ -14,15 +14,21 @@ class kgramstats
 {
 public:
         kgramstats(string corpus, int maxK);
-        map<string, int>* lookupExts(kgram tk);
-        int getMaxK();
         vector<string> randomSentence(int n);
         
 private:
+        typedef struct
+        {
+                int all;
+                int titlecase;
+                int uppercase;
+                int period;
+        } token_data;
         int maxK;
-        map<kgram, map<string, int>* >* stats;
+        map<kgram, map<string, token_data*>* >* stats;
 };
 
 void printKgram(kgram k);
+std::string canonize(std::string f);
 
 #endif
\ No newline at end of file