diff options
Diffstat (limited to 'generator/generator.cpp')
-rw-r--r-- | generator/generator.cpp | 21 |
1 files changed, 17 insertions, 4 deletions
diff --git a/generator/generator.cpp b/generator/generator.cpp index e52aa90..0d073be 100644 --- a/generator/generator.cpp +++ b/generator/generator.cpp | |||
@@ -696,7 +696,8 @@ namespace verbly { | |||
696 | 696 | ||
697 | void generator::readWordNetAntonymy() | 697 | void generator::readWordNetAntonymy() |
698 | { | 698 | { |
699 | std::list<std::string> lines(readFile(wordNetPath_ + "wn_ant.pl")); | 699 | std::list<std::string> lines(readFile(wordNetPath_ + "wn_ant.pl", true)); |
700 | |||
700 | hatkirby::progress ppgs("Writing antonyms...", lines.size()); | 701 | hatkirby::progress ppgs("Writing antonyms...", lines.size()); |
701 | for (auto line : lines) | 702 | for (auto line : lines) |
702 | { | 703 | { |
@@ -770,7 +771,7 @@ namespace verbly { | |||
770 | 771 | ||
771 | void generator::readWordNetClasses() | 772 | void generator::readWordNetClasses() |
772 | { | 773 | { |
773 | std::list<std::string> lines(readFile(wordNetPath_ + "wn_cls.pl")); | 774 | std::list<std::string> lines(readFile(wordNetPath_ + "wn_cls.pl", true)); |
774 | 775 | ||
775 | hatkirby::progress ppgs( | 776 | hatkirby::progress ppgs( |
776 | "Writing usage, topicality, and regionality...", | 777 | "Writing usage, topicality, and regionality...", |
@@ -1092,7 +1093,7 @@ namespace verbly { | |||
1092 | 1093 | ||
1093 | void generator::readWordNetPertainymy() | 1094 | void generator::readWordNetPertainymy() |
1094 | { | 1095 | { |
1095 | std::list<std::string> lines(readFile(wordNetPath_ + "wn_per.pl")); | 1096 | std::list<std::string> lines(readFile(wordNetPath_ + "wn_per.pl", true)); |
1096 | 1097 | ||
1097 | hatkirby::progress ppgs( | 1098 | hatkirby::progress ppgs( |
1098 | "Writing pertainymy and mannernymy...", | 1099 | "Writing pertainymy and mannernymy...", |
@@ -1228,7 +1229,7 @@ namespace verbly { | |||
1228 | db_.execute("ANALYZE"); | 1229 | db_.execute("ANALYZE"); |
1229 | } | 1230 | } |
1230 | 1231 | ||
1231 | std::list<std::string> generator::readFile(std::string path) | 1232 | std::list<std::string> generator::readFile(std::string path, bool uniq) |
1232 | { | 1233 | { |
1233 | std::ifstream file(path); | 1234 | std::ifstream file(path); |
1234 | if (!file) | 1235 | if (!file) |
@@ -1248,6 +1249,18 @@ namespace verbly { | |||
1248 | lines.push_back(line); | 1249 | lines.push_back(line); |
1249 | } | 1250 | } |
1250 | 1251 | ||
1252 | if (uniq) | ||
1253 | { | ||
1254 | std::vector<std::string> uniq(std::begin(lines), std::end(lines)); | ||
1255 | lines.clear(); | ||
1256 | |||
1257 | std::sort(std::begin(uniq), std::end(uniq)); | ||
1258 | std::unique_copy( | ||
1259 | std::begin(uniq), | ||
1260 | std::end(uniq), | ||
1261 | std::back_inserter(lines)); | ||
1262 | } | ||
1263 | |||
1251 | return lines; | 1264 | return lines; |
1252 | } | 1265 | } |
1253 | 1266 | ||