From 420a7a1e004410f1377a6d919d72d18f8ae34bdf Mon Sep 17 00:00:00 2001
From: Feffernoose <fefferburbia@gmail.com>
Date: Tue, 1 Oct 2013 21:29:15 -0400
Subject: Weighed token casing and presence of periods

Tokens which differ only by casing or the presence of an ending period are
now considered the same token. When tokens are generated, they are cased
based on the prevalence of Upper/Title/Lower casing of the token in the
input corpus, and similarly, a period is added to the end of the word based
on how often the same token was ended with a period in the input corpus.
---
 kgramstats.cpp | 92 ++++++++++++++++++++++++++++++++++++++++++----------------
 kgramstats.h   | 12 ++++++--
 2 files changed, 76 insertions(+), 28 deletions(-)

diff --git a/kgramstats.cpp b/kgramstats.cpp
index 142b5aa..708013f 100644
--- a/kgramstats.cpp
+++ b/kgramstats.cpp
@@ -2,6 +2,7 @@
 #include <vector>
 #include <iostream>
 #include <cstdlib>
+#include <algorithm>
 
 kgramstats::kgramstats(string corpus, int maxK)
 {
@@ -20,34 +21,45 @@ kgramstats::kgramstats(string corpus, int maxK)
 	   start = ((end > (string::npos - 1) ) ? string::npos : end + 1);
 	}
 	
-	stats = new map<kgram, map<string, int>* >();
+	stats = new map<kgram, map<string, token_data*>* >();
 	for (int k=0; k<=maxK; k++)
 	{
 		for (int i=0; i<(tokens.size() - k); i++)
 		{
 			kgram seq(tokens.begin()+i, tokens.begin()+i+k);
+			transform(seq.begin(), seq.end(), seq.begin(), canonize);
 			string f = tokens[i+k];
+			string canonical = canonize(f);
 			
 			if ((*stats)[seq] == NULL)
 			{
-				(*stats)[seq] = new map<string, int>();
+				(*stats)[seq] = new map<string, token_data*>();
 			}
 			
-			(*((*stats)[seq]))[f]++;
+			if ((*(*stats)[seq])[canonical] == NULL)
+			{
+				(*(*stats)[seq])[canonical] = (token_data*) calloc(1, sizeof(token_data));
+			}
+
+			token_data* td = stats->at(seq)->at(canonical);
+			td->all++;
+			
+			if ((f.length() > 0) && (f[f.length()-1] == '.'))
+			{
+				td->period++;
+			}
+			
+			if (std::find_if(f.begin(), f.end(), ::islower) == f.end())
+			{
+				td->uppercase++;
+			} else if (isupper(f[0]))
+			{
+				td->titlecase++;
+			}
 		}
 	}
 }
 
-map<string, int>* kgramstats::lookupExts(kgram tk)
-{
-	return (*stats)[tk];
-}
-
-int kgramstats::getMaxK()
-{
-	return maxK;
-}
-
 void printKgram(kgram k)
 {
 	for (kgram::iterator it = k.begin(); it != k.end(); it++)
@@ -76,35 +88,65 @@ vector<string> kgramstats::randomSentence(int n)
 				}
 			}
 		}
-		
-		map<string, int>* probtable = lookupExts(cur);
+
+		map<string, token_data*>* probtable = (*stats)[cur];
 		int max = 0;
-		for (map<string, int>::iterator it = probtable->begin(); it != probtable->end(); ++it)
+		for (map<string, token_data*>::iterator it = probtable->begin(); it != probtable->end(); ++it)
 		{
-			max += it->second;
+			max += it->second->all;
 		}
-		
+
 		int r = rand() % (max+1);
-		string next = probtable->begin()->first;
-		for (map<string, int>::iterator it = probtable->begin(); it != probtable->end(); ++it)
+		map<string, token_data*>::iterator next = probtable->begin();
+		for (map<string, token_data*>::iterator it = probtable->begin(); it != probtable->end(); ++it)
 		{
-			if (it->second > r)
+			if (it->second->all > r)
 			{
 				break;
 			} else {
-				next = it->first;
-				r -= it->second;
+				next = it;
+				r -= it->second->all;
 			}
 		}
 
+		string nextToken(next->first);
+		int casing = rand() % next->second->all;
+		int period = rand() % next->second->all;
+		if (casing < next->second->uppercase)
+		{
+			transform(nextToken.begin(), nextToken.end(), nextToken.begin(), ::toupper);
+		} else if ((casing - next->second->uppercase) < next->second->titlecase)
+		{
+			nextToken[0] = toupper(nextToken[0]);
+		}
+		
+		if (period < next->second->period)
+		{
+			nextToken += ".";
+		}
+		
+		cout << next->first << " | " << nextToken << endl;
+
 		if (cur.size() == maxK)
 		{
 			cur.pop_front();
 		}
 
-		cur.push_back(next);
-		result.push_back(next);
+		cur.push_back(next->first);
+		result.push_back(nextToken);
 	}
 	
 	return result;
+}
+
+std::string canonize(std::string f)
+{
+	string canonical(f);
+	transform(canonical.begin(), canonical.end(), canonical.begin(), ::tolower);
+	if (canonical[canonical.length()-1] == '.')
+	{
+		canonical.resize(canonical.find('.'));
+	}
+	
+	return canonical;
 }
\ No newline at end of file
diff --git a/kgramstats.h b/kgramstats.h
index 069bb90..248b193 100644
--- a/kgramstats.h
+++ b/kgramstats.h
@@ -14,15 +14,21 @@ class kgramstats
 {
 public:
 	kgramstats(string corpus, int maxK);
-	map<string, int>* lookupExts(kgram tk);
-	int getMaxK();
 	vector<string> randomSentence(int n);
 	
 private:
+	typedef struct
+	{
+		int all;
+		int titlecase;
+		int uppercase;
+		int period;
+	} token_data;
 	int maxK;
-	map<kgram, map<string, int>* >* stats;
+	map<kgram, map<string, token_data*>* >* stats;
 };
 
 void printKgram(kgram k);
+std::string canonize(std::string f);
 
 #endif
\ No newline at end of file
-- 
cgit 1.4.1