about summary refs log tree commit diff stats
diff options
context:
space:
mode:
authorFeffernoose <fefferburbia@gmail.com>2013-10-06 15:03:29 -0400
committerFeffernoose <fefferburbia@gmail.com>2013-10-06 15:03:29 -0400
commit8e2693ebb6670f8b8c66f5818be87140467ed5f3 (patch)
tree6f78c3cd5cee959fbed0101ea96531513bfadb93
parenteb076ca2c6c8932fd251419563cf0078c5ee0914 (diff)
downloadrawr-ebooks-8e2693ebb6670f8b8c66f5818be87140467ed5f3.tar.gz
rawr-ebooks-8e2693ebb6670f8b8c66f5818be87140467ed5f3.tar.bz2
rawr-ebooks-8e2693ebb6670f8b8c66f5818be87140467ed5f3.zip
Stripped empty tokens from corpus
-rw-r--r--kgramstats.cpp10
1 files changed, 8 insertions, 2 deletions
diff --git a/kgramstats.cpp b/kgramstats.cpp index 6c0e4ce..327752b 100644 --- a/kgramstats.cpp +++ b/kgramstats.cpp
@@ -4,6 +4,8 @@
4#include <cstdlib> 4#include <cstdlib>
5#include <algorithm> 5#include <algorithm>
6 6
7// runs in O(t^2) time where t is the number of tokens in the input corpus
8// We consider maxK to be fairly constant
7kgramstats::kgramstats(string corpus, int maxK) 9kgramstats::kgramstats(string corpus, int maxK)
8{ 10{
9 this->maxK = maxK; 11 this->maxK = maxK;
@@ -16,7 +18,11 @@ kgramstats::kgramstats(string corpus, int maxK)
16 { 18 {
17 end = corpus.find(" ", start); 19 end = corpus.find(" ", start);
18 20
19 tokens.push_back(corpus.substr(start, (end == string::npos) ? string::npos : end - start)); 21 string token = corpus.substr(start, (end == string::npos) ? string::npos : end - start);
22 if (token.compare(""))
23 {
24 tokens.push_back(token);
25 }
20 26
21 start = ((end > (string::npos - 1) ) ? string::npos : end + 1); 27 start = ((end > (string::npos - 1) ) ? string::npos : end + 1);
22 } 28 }
@@ -85,9 +91,9 @@ void printKgram(kgram k)
85 { 91 {
86 cout << *it << " "; 92 cout << *it << " ";
87 } 93 }
88 cout << endl;
89} 94}
90 95
96// runs in O(n log t) time where n is the input number of sentences and t is the number of tokens in the input corpus
91vector<string> kgramstats::randomSentence(int n) 97vector<string> kgramstats::randomSentence(int n)
92{ 98{
93 vector<string> result; 99 vector<string> result;