From 420a7a1e004410f1377a6d919d72d18f8ae34bdf Mon Sep 17 00:00:00 2001
From: Feffernoose <fefferburbia@gmail.com>
Date: Tue, 1 Oct 2013 21:29:15 -0400
Subject: Weighed token casing and presence of periods

Tokens which differ only by casing or the presence of an ending period are
now considered the same token. When tokens are generated, they are cased
based on the prevalence of Upper/Title/Lower casing of the token in the
input corpus, and similarly, a period is added to the end of the word based
on how often the same token was ended with a period in the input corpus.
---
 kgramstats.h | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

(limited to 'kgramstats.h')

diff --git a/kgramstats.h b/kgramstats.h
index 069bb90..248b193 100644
--- a/kgramstats.h
+++ b/kgramstats.h
@@ -14,15 +14,21 @@ class kgramstats
 {
 public:
 	kgramstats(string corpus, int maxK);
-	map<string, int>* lookupExts(kgram tk);
-	int getMaxK();
 	vector<string> randomSentence(int n);
 	
 private:
+	typedef struct
+	{
+		int all;
+		int titlecase;
+		int uppercase;
+		int period;
+	} token_data;
 	int maxK;
-	map<kgram, map<string, int>* >* stats;
+	map<kgram, map<string, token_data*>* >* stats;
 };
 
 void printKgram(kgram k);
+std::string canonize(std::string f);
 
 #endif
\ No newline at end of file
-- 
cgit 1.4.1