diff options
Diffstat (limited to 'verbly/generator/generator.cpp')
-rw-r--r-- | verbly/generator/generator.cpp | 1663 |
1 files changed, 0 insertions, 1663 deletions
diff --git a/verbly/generator/generator.cpp b/verbly/generator/generator.cpp deleted file mode 100644 index faef5f7..0000000 --- a/verbly/generator/generator.cpp +++ /dev/null | |||
@@ -1,1663 +0,0 @@ | |||
1 | #include <libxml/parser.h> | ||
2 | #include <iostream> | ||
3 | #include <dirent.h> | ||
4 | #include <set> | ||
5 | #include <map> | ||
6 | #include <string> | ||
7 | #include <vector> | ||
8 | #include <fstream> | ||
9 | #include <sqlite3.h> | ||
10 | #include <sstream> | ||
11 | #include <regex> | ||
12 | #include <list> | ||
13 | #include "progress.h" | ||
14 | |||
15 | struct verb { | ||
16 | std::string infinitive; | ||
17 | std::string past_tense; | ||
18 | std::string past_participle; | ||
19 | std::string ing_form; | ||
20 | std::string s_form; | ||
21 | }; | ||
22 | |||
23 | struct adjective { | ||
24 | std::string base; | ||
25 | std::string comparative; | ||
26 | std::string superlative; | ||
27 | }; | ||
28 | |||
29 | struct noun { | ||
30 | std::string singular; | ||
31 | std::string plural; | ||
32 | }; | ||
33 | |||
34 | struct group { | ||
35 | std::string id; | ||
36 | std::set<std::string> members; | ||
37 | }; | ||
38 | |||
39 | std::map<std::string, group> groups; | ||
40 | std::map<std::string, verb> verbs; | ||
41 | std::map<std::string, adjective> adjectives; | ||
42 | std::map<std::string, noun> nouns; | ||
43 | std::map<int, std::map<int, int>> wn; | ||
44 | std::map<std::string, std::set<std::string>> pronunciations; | ||
45 | |||
46 | void print_usage() | ||
47 | { | ||
48 | std::cout << "Verbly Datafile Generator" << std::endl; | ||
49 | std::cout << "-------------------------" << std::endl; | ||
50 | std::cout << "Requires exactly six arguments." << std::endl; | ||
51 | std::cout << "1. The path to a VerbNet data directory." << std::endl; | ||
52 | std::cout << "2. The path to a SemLink vnpbMappings file." << std::endl; | ||
53 | std::cout << "3. The path to an AGID infl.txt file." << std::endl; | ||
54 | std::cout << "4. The path to a WordNet prolog data directory." << std::endl; | ||
55 | std::cout << "5. The path to a CMUDICT pronunciation file." << std::endl; | ||
56 | std::cout << "6. Datafile output path." << std::endl; | ||
57 | |||
58 | exit(1); | ||
59 | } | ||
60 | |||
61 | void db_error(sqlite3* ppdb, std::string) | ||
62 | { | ||
63 | std::cout << "Error writing to output database: " << sqlite3_errmsg(ppdb) << std::endl; | ||
64 | sqlite3_close_v2(ppdb); | ||
65 | print_usage(); | ||
66 | } | ||
67 | |||
68 | /* | ||
69 | void parse_group(xmlNodePtr top, std::string filename) | ||
70 | { | ||
71 | xmlChar* key = xmlGetProp(top, (xmlChar*) "ID"); | ||
72 | if (key == 0) | ||
73 | { | ||
74 | std::cout << "Bad VerbNet file format: " << filename << std::endl; | ||
75 | print_usage(); | ||
76 | } | ||
77 | std::string vnid = key; | ||
78 | vnid = vnid.substr(vnid.find_first_of("-")+1); | ||
79 | xmlFree(key); | ||
80 | |||
81 | group g; | ||
82 | g.id = vnid; | ||
83 | |||
84 | for (xmlNodePtr node = top->xmlChildrenNode; node != nullptr; node = node->next) | ||
85 | { | ||
86 | if (!xmlStrcmp(node->name, (const xmlChar*) "MEMBERS")) | ||
87 | { | ||
88 | for (xmlNodePtr member = node->xmlChildrenNode; member != nullptr; member = member->next) | ||
89 | { | ||
90 | if (!xmlStrcmp(member->name, (const xmlChar*) "MEMBER")) | ||
91 | { | ||
92 | key = xmlGetProp(member, (xmlChar*) "name"); | ||
93 | g.members.insert(key); | ||
94 | xmlFree(key); | ||
95 | } | ||
96 | } | ||
97 | } else if (!xmlStrcmp(node->name, (const xmlChar*) "FRAMES")) | ||
98 | { | ||
99 | for (xmlNodePtr frame = node->xmlChildrenNode; frame != nullptr; frame = frame->next) | ||
100 | { | ||
101 | if (!xmlStrcmp(frame->name, (const xmlChar*) "FRAME")) | ||
102 | { | ||
103 | for (xmlNodePtr framenode = frame->xmlChildrenNode; framenode != nullptr; framenode = framenode->next) | ||
104 | { | ||
105 | |||
106 | } | ||
107 | } | ||
108 | } | ||
109 | } | ||
110 | } | ||
111 | }*/ | ||
112 | |||
113 | int main(int argc, char** argv) | ||
114 | { | ||
115 | if (argc != 7) | ||
116 | { | ||
117 | print_usage(); | ||
118 | } | ||
119 | |||
120 | /*DIR* dir; | ||
121 | if ((dir = opendir(argv[1])) == nullptr) | ||
122 | { | ||
123 | std::cout << "Invalid VerbNet data directory." << std::endl; | ||
124 | |||
125 | print_usage(); | ||
126 | } | ||
127 | |||
128 | struct dirent* ent; | ||
129 | while ((ent = readdir(dir)) != nullptr) | ||
130 | { | ||
131 | std::string filename(argv[1]); | ||
132 | if (filename.back() != '/') | ||
133 | { | ||
134 | filename += '/'; | ||
135 | } | ||
136 | |||
137 | filename += ent->d_name; | ||
138 | //std::cout << ent->d_name << std::endl; | ||
139 | |||
140 | if (filename.rfind(".xml") != filename.size() - 4) | ||
141 | { | ||
142 | continue; | ||
143 | } | ||
144 | |||
145 | xmlDocPtr doc = xmlParseFile(filename.c_str()); | ||
146 | if (doc == nullptr) | ||
147 | { | ||
148 | std::cout << "Error opening " << filename << std::endl; | ||
149 | print_usage(); | ||
150 | } | ||
151 | |||
152 | xmlNodePtr top = xmlDocGetRootElement(doc); | ||
153 | if ((top == nullptr) || (xmlStrcmp(top->name, (xmlChar*) "VNCLASS"))) | ||
154 | { | ||
155 | std::cout << "Bad VerbNet file format: " << filename << std::endl; | ||
156 | print_usage(); | ||
157 | } | ||
158 | |||
159 | parse_group(top, filename); | ||
160 | } | ||
161 | |||
162 | closedir(dir);*/ | ||
163 | |||
164 | // Get verbs from AGID | ||
165 | std::cout << "Reading inflections..." << std::endl; | ||
166 | |||
167 | std::ifstream agidfile(argv[3]); | ||
168 | if (!agidfile.is_open()) | ||
169 | { | ||
170 | std::cout << "Could not open AGID file: " << argv[3] << std::endl; | ||
171 | print_usage(); | ||
172 | } | ||
173 | |||
174 | for (;;) | ||
175 | { | ||
176 | std::string line; | ||
177 | if (!getline(agidfile, line)) | ||
178 | { | ||
179 | break; | ||
180 | } | ||
181 | |||
182 | if (line.back() == '\r') | ||
183 | { | ||
184 | line.pop_back(); | ||
185 | } | ||
186 | |||
187 | int divider = line.find_first_of(" "); | ||
188 | std::string word = line.substr(0, divider); | ||
189 | line = line.substr(divider+1); | ||
190 | char type = line[0]; | ||
191 | |||
192 | if (line[1] == '?') | ||
193 | { | ||
194 | line.erase(0, 4); | ||
195 | } else { | ||
196 | line.erase(0, 3); | ||
197 | } | ||
198 | |||
199 | std::vector<std::string> forms; | ||
200 | while (!line.empty()) | ||
201 | { | ||
202 | std::string inflection; | ||
203 | if ((divider = line.find(" | ")) != std::string::npos) | ||
204 | { | ||
205 | inflection = line.substr(0, divider); | ||
206 | line = line.substr(divider + 3); | ||
207 | } else { | ||
208 | inflection = line; | ||
209 | line = ""; | ||
210 | } | ||
211 | |||
212 | if ((divider = inflection.find_first_of(",?")) != std::string::npos) | ||
213 | { | ||
214 | inflection = inflection.substr(0, divider); | ||
215 | } | ||
216 | |||
217 | forms.push_back(inflection); | ||
218 | } | ||
219 | |||
220 | switch (type) | ||
221 | { | ||
222 | case 'V': | ||
223 | { | ||
224 | verb v; | ||
225 | v.infinitive = word; | ||
226 | if (forms.size() == 4) | ||
227 | { | ||
228 | v.past_tense = forms[0]; | ||
229 | v.past_participle = forms[1]; | ||
230 | v.ing_form = forms[2]; | ||
231 | v.s_form = forms[3]; | ||
232 | } else if (forms.size() == 3) | ||
233 | { | ||
234 | v.past_tense = forms[0]; | ||
235 | v.past_participle = forms[0]; | ||
236 | v.ing_form = forms[1]; | ||
237 | v.s_form = forms[2]; | ||
238 | } else if (forms.size() == 8) | ||
239 | { | ||
240 | // As of AGID 2014.08.11, this is only "to be" | ||
241 | v.past_tense = forms[0]; | ||
242 | v.past_participle = forms[2]; | ||
243 | v.ing_form = forms[3]; | ||
244 | v.s_form = forms[4]; | ||
245 | } else { | ||
246 | // Words that don't fit the cases above as of AGID 2014.08.11: | ||
247 | // - may and shall do not conjugate the way we want them to | ||
248 | // - methinks only has a past tense and is an outlier | ||
249 | // - wit has five forms, and is archaic/obscure enough that we can ignore it for now | ||
250 | std::cout << "Ignoring verb \"" << word << "\" due to non-standard number of forms." << std::endl; | ||
251 | } | ||
252 | |||
253 | verbs[word] = v; | ||
254 | |||
255 | break; | ||
256 | } | ||
257 | |||
258 | case 'A': | ||
259 | { | ||
260 | adjective adj; | ||
261 | adj.base = word; | ||
262 | if (forms.size() == 2) | ||
263 | { | ||
264 | adj.comparative = forms[0]; | ||
265 | adj.superlative = forms[1]; | ||
266 | } else { | ||
267 | // As of AGID 2014.08.11, this is only "only", which has only the form "onliest" | ||
268 | std::cout << "Ignoring adjective/adverb \"" << word << "\" due to non-standard number of forms." << std::endl; | ||
269 | } | ||
270 | |||
271 | adjectives[word] = adj; | ||
272 | |||
273 | break; | ||
274 | } | ||
275 | |||
276 | case 'N': | ||
277 | { | ||
278 | noun n; | ||
279 | n.singular = word; | ||
280 | if (forms.size() == 1) | ||
281 | { | ||
282 | n.plural = forms[0]; | ||
283 | } else { | ||
284 | // As of AGID 2014.08.11, this is non-existent. | ||
285 | std::cout << "Ignoring noun \"" << word << "\" due to non-standard number of forms." << std::endl; | ||
286 | } | ||
287 | |||
288 | nouns[word] = n; | ||
289 | |||
290 | break; | ||
291 | } | ||
292 | } | ||
293 | } | ||
294 | |||
295 | // Pronounciations | ||
296 | std::cout << "Reading pronunciations..." << std::endl; | ||
297 | |||
298 | std::ifstream pronfile(argv[5]); | ||
299 | if (!pronfile.is_open()) | ||
300 | { | ||
301 | std::cout << "Could not open CMUDICT file: " << argv[5] << std::endl; | ||
302 | print_usage(); | ||
303 | } | ||
304 | |||
305 | for (;;) | ||
306 | { | ||
307 | std::string line; | ||
308 | if (!getline(pronfile, line)) | ||
309 | { | ||
310 | break; | ||
311 | } | ||
312 | |||
313 | if (line.back() == '\r') | ||
314 | { | ||
315 | line.pop_back(); | ||
316 | } | ||
317 | |||
318 | std::regex phoneme("([A-Z][^ \\(]*)(?:\\(\\d+\\))? ([A-Z 0-9]+)"); | ||
319 | std::smatch phoneme_data; | ||
320 | if (std::regex_search(line, phoneme_data, phoneme)) | ||
321 | { | ||
322 | std::string canonical(phoneme_data[1]); | ||
323 | std::transform(std::begin(canonical), std::end(canonical), std::begin(canonical), ::tolower); | ||
324 | |||
325 | pronunciations[canonical].insert(phoneme_data[2]); | ||
326 | } | ||
327 | } | ||
328 | |||
329 | // Start writing output | ||
330 | std::cout << "Writing schema..." << std::endl; | ||
331 | |||
332 | sqlite3* ppdb; | ||
333 | if (sqlite3_open_v2(argv[6], &ppdb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL) != SQLITE_OK) | ||
334 | { | ||
335 | std::cout << "Error opening output datafile: " << sqlite3_errmsg(ppdb) << std::endl; | ||
336 | print_usage(); | ||
337 | } | ||
338 | |||
339 | std::ifstream schemafile("schema.sql"); | ||
340 | if (!schemafile.is_open()) | ||
341 | { | ||
342 | std::cout << "Could not find schema file" << std::endl; | ||
343 | print_usage(); | ||
344 | } | ||
345 | |||
346 | std::stringstream schemabuilder; | ||
347 | for (;;) | ||
348 | { | ||
349 | std::string line; | ||
350 | if (!getline(schemafile, line)) | ||
351 | { | ||
352 | break; | ||
353 | } | ||
354 | |||
355 | if (line.back() == '\r') | ||
356 | { | ||
357 | line.pop_back(); | ||
358 | } | ||
359 | |||
360 | schemabuilder << line << std::endl; | ||
361 | } | ||
362 | |||
363 | std::string schema = schemabuilder.str(); | ||
364 | while (!schema.empty()) | ||
365 | { | ||
366 | std::string query; | ||
367 | int divider = schema.find(";"); | ||
368 | if (divider != std::string::npos) | ||
369 | { | ||
370 | query = schema.substr(0, divider+1); | ||
371 | schema = schema.substr(divider+2); | ||
372 | } else { | ||
373 | break; | ||
374 | } | ||
375 | |||
376 | sqlite3_stmt* schmstmt; | ||
377 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &schmstmt, NULL) != SQLITE_OK) | ||
378 | { | ||
379 | db_error(ppdb, query); | ||
380 | } | ||
381 | |||
382 | if (sqlite3_step(schmstmt) != SQLITE_DONE) | ||
383 | { | ||
384 | db_error(ppdb, query); | ||
385 | } | ||
386 | |||
387 | sqlite3_finalize(schmstmt); | ||
388 | } | ||
389 | |||
390 | { | ||
391 | progress ppgs("Writing verbs...", verbs.size()); | ||
392 | for (auto& mapping : verbs) | ||
393 | { | ||
394 | sqlite3_stmt* ppstmt; | ||
395 | std::string query("INSERT INTO verbs (infinitive, past_tense, past_participle, ing_form, s_form) VALUES (?, ?, ?, ?, ?)"); | ||
396 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
397 | { | ||
398 | db_error(ppdb, query); | ||
399 | } | ||
400 | |||
401 | sqlite3_bind_text(ppstmt, 1, mapping.second.infinitive.c_str(), mapping.second.infinitive.length(), SQLITE_STATIC); | ||
402 | sqlite3_bind_text(ppstmt, 2, mapping.second.past_tense.c_str(), mapping.second.past_tense.length(), SQLITE_STATIC); | ||
403 | sqlite3_bind_text(ppstmt, 3, mapping.second.past_participle.c_str(), mapping.second.past_participle.length(), SQLITE_STATIC); | ||
404 | sqlite3_bind_text(ppstmt, 4, mapping.second.ing_form.c_str(), mapping.second.ing_form.length(), SQLITE_STATIC); | ||
405 | sqlite3_bind_text(ppstmt, 5, mapping.second.s_form.c_str(), mapping.second.s_form.length(), SQLITE_STATIC); | ||
406 | |||
407 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
408 | { | ||
409 | db_error(ppdb, query); | ||
410 | } | ||
411 | |||
412 | sqlite3_finalize(ppstmt); | ||
413 | |||
414 | std::string canonical(mapping.second.infinitive); | ||
415 | std::transform(std::begin(canonical), std::end(canonical), std::begin(canonical), ::tolower); | ||
416 | if (pronunciations.count(canonical) == 1) | ||
417 | { | ||
418 | query = "SELECT last_insert_rowid()"; | ||
419 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
420 | { | ||
421 | db_error(ppdb, query); | ||
422 | } | ||
423 | |||
424 | if (sqlite3_step(ppstmt) != SQLITE_ROW) | ||
425 | { | ||
426 | db_error(ppdb, query); | ||
427 | } | ||
428 | |||
429 | int rowid = sqlite3_column_int(ppstmt, 0); | ||
430 | |||
431 | sqlite3_finalize(ppstmt); | ||
432 | |||
433 | for (auto pronunciation : pronunciations[canonical]) | ||
434 | { | ||
435 | query = "INSERT INTO verb_pronunciations (verb_id, pronunciation) VALUES (?, ?)"; | ||
436 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
437 | { | ||
438 | db_error(ppdb, query); | ||
439 | } | ||
440 | |||
441 | sqlite3_bind_int(ppstmt, 1, rowid); | ||
442 | sqlite3_bind_text(ppstmt, 2, pronunciation.c_str(), pronunciation.length(), SQLITE_STATIC); | ||
443 | |||
444 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
445 | { | ||
446 | db_error(ppdb, query); | ||
447 | } | ||
448 | |||
449 | sqlite3_finalize(ppstmt); | ||
450 | } | ||
451 | } | ||
452 | |||
453 | ppgs.update(); | ||
454 | } | ||
455 | } | ||
456 | |||
457 | // Get nouns/adjectives/adverbs from WordNet | ||
458 | // Useful relations: | ||
459 | // - s: master list | ||
460 | // - ant: antonymy (e.g. happy/sad, sad/happy, happiness/sadness) | ||
461 | // - at: variation (e.g. a measurement can be standard or nonstandard) | ||
462 | // - der: derivation (e.g. happy/happily, happily/happy) | ||
463 | // - hyp: hypernymy/hyponymy (e.g. color/red, color/blue) | ||
464 | // - ins: instantiation (do we need this? let's see) | ||
465 | // - mm: member meronymy/holonymy (e.g. family/mother, family/child) | ||
466 | // - mp: part meronymy/holonymy (e.g. wheel/spoke, wheel/tire) | ||
467 | // - ms: substance meronymy/holonymy (e.g. tire/rubber, doorstop/rubber) | ||
468 | // - per: pertainymy (e.g. something that is Alaskan pertains to Alaska) | ||
469 | // mannernymy (e.g. something done quickly is done in a manner that is quick) | ||
470 | // - sa: specification (e.g. inaccurate (general) can mean imprecise or incorrect (specific)) | ||
471 | // - sim: synonymy (e.g. cheerful/happy, happy/cheerful) | ||
472 | // - syntax: positioning flags for some adjectives | ||
473 | std::string wnpref {argv[4]}; | ||
474 | if (wnpref.back() != '/') | ||
475 | { | ||
476 | wnpref += '/'; | ||
477 | } | ||
478 | |||
479 | // s table | ||
480 | { | ||
481 | std::ifstream wnsfile(wnpref + "wn_s.pl"); | ||
482 | if (!wnsfile.is_open()) | ||
483 | { | ||
484 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
485 | print_usage(); | ||
486 | } | ||
487 | |||
488 | std::list<std::string> lines; | ||
489 | for (;;) | ||
490 | { | ||
491 | std::string line; | ||
492 | if (!getline(wnsfile, line)) | ||
493 | { | ||
494 | break; | ||
495 | } | ||
496 | |||
497 | if (line.back() == '\r') | ||
498 | { | ||
499 | line.pop_back(); | ||
500 | } | ||
501 | |||
502 | lines.push_back(line); | ||
503 | } | ||
504 | |||
505 | progress ppgs("Writing nouns, adjectives, and adverbs...", lines.size()); | ||
506 | for (auto line : lines) | ||
507 | { | ||
508 | ppgs.update(); | ||
509 | |||
510 | std::regex relation("^s\\(([134]\\d{8}),(\\d+),'([\\w ]+)',"); | ||
511 | std::smatch relation_data; | ||
512 | if (!std::regex_search(line, relation_data, relation)) | ||
513 | { | ||
514 | continue; | ||
515 | } | ||
516 | |||
517 | int synset_id = stoi(relation_data[1]); | ||
518 | int wnum = stoi(relation_data[2]); | ||
519 | std::string word = relation_data[3]; | ||
520 | |||
521 | std::string query; | ||
522 | switch (synset_id / 100000000) | ||
523 | { | ||
524 | case 1: // Noun | ||
525 | { | ||
526 | if (nouns.count(word) == 1) | ||
527 | { | ||
528 | query = "INSERT INTO nouns (singular, plural) VALUES (?, ?)"; | ||
529 | } else { | ||
530 | query = "INSERT INTO nouns (singular) VALUES (?)"; | ||
531 | } | ||
532 | |||
533 | break; | ||
534 | } | ||
535 | |||
536 | case 2: // Verb | ||
537 | { | ||
538 | // Ignore | ||
539 | |||
540 | break; | ||
541 | } | ||
542 | |||
543 | case 3: // Adjective | ||
544 | { | ||
545 | if (adjectives.count(word) == 1) | ||
546 | { | ||
547 | query = "INSERT INTO adjectives (base_form, comparative, superlative) VALUES (?, ?, ?)"; | ||
548 | } else { | ||
549 | query = "INSERT INTO adjectives (base_form) VALUES (?)"; | ||
550 | } | ||
551 | |||
552 | break; | ||
553 | } | ||
554 | |||
555 | case 4: // Adverb | ||
556 | { | ||
557 | if (adjectives.count(word) == 1) | ||
558 | { | ||
559 | query = "INSERT INTO adverbs (base_form, comparative, superlative) VALUES (?, ?, ?)"; | ||
560 | } else { | ||
561 | query = "INSERT INTO adverbs (base_form) VALUES (?)"; | ||
562 | } | ||
563 | |||
564 | break; | ||
565 | } | ||
566 | } | ||
567 | |||
568 | sqlite3_stmt* ppstmt; | ||
569 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
570 | { | ||
571 | db_error(ppdb, query); | ||
572 | } | ||
573 | |||
574 | sqlite3_bind_text(ppstmt, 1, word.c_str(), word.length(), SQLITE_STATIC); | ||
575 | switch (synset_id / 100000000) | ||
576 | { | ||
577 | case 1: // Noun | ||
578 | { | ||
579 | if (nouns.count(word) == 1) | ||
580 | { | ||
581 | sqlite3_bind_text(ppstmt, 2, nouns[word].plural.c_str(), nouns[word].plural.length(), SQLITE_STATIC); | ||
582 | } | ||
583 | |||
584 | break; | ||
585 | } | ||
586 | |||
587 | case 3: // Adjective | ||
588 | case 4: // Adverb | ||
589 | { | ||
590 | if (adjectives.count(word) == 1) | ||
591 | { | ||
592 | sqlite3_bind_text(ppstmt, 2, adjectives[word].comparative.c_str(), adjectives[word].comparative.length(), SQLITE_STATIC); | ||
593 | sqlite3_bind_text(ppstmt, 3, adjectives[word].superlative.c_str(), adjectives[word].superlative.length(), SQLITE_STATIC); | ||
594 | } | ||
595 | |||
596 | break; | ||
597 | } | ||
598 | } | ||
599 | |||
600 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
601 | { | ||
602 | db_error(ppdb, query); | ||
603 | } | ||
604 | |||
605 | sqlite3_finalize(ppstmt); | ||
606 | |||
607 | query = "SELECT last_insert_rowid()"; | ||
608 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
609 | { | ||
610 | db_error(ppdb, query); | ||
611 | } | ||
612 | |||
613 | if (sqlite3_step(ppstmt) != SQLITE_ROW) | ||
614 | { | ||
615 | db_error(ppdb, query); | ||
616 | } | ||
617 | |||
618 | int rowid = sqlite3_column_int(ppstmt, 0); | ||
619 | wn[synset_id][wnum] = rowid; | ||
620 | |||
621 | sqlite3_finalize(ppstmt); | ||
622 | |||
623 | std::string canonical(word); | ||
624 | std::transform(std::begin(canonical), std::end(canonical), std::begin(canonical), ::tolower); | ||
625 | if (pronunciations.count(canonical) == 1) | ||
626 | { | ||
627 | for (auto pronunciation : pronunciations[canonical]) | ||
628 | { | ||
629 | switch (synset_id / 100000000) | ||
630 | { | ||
631 | case 1: // Noun | ||
632 | { | ||
633 | query = "INSERT INTO noun_pronunciations (noun_id, pronunciation) VALUES (?, ?)"; | ||
634 | |||
635 | break; | ||
636 | } | ||
637 | |||
638 | case 3: // Adjective | ||
639 | { | ||
640 | query = "INSERT INTO adjective_pronunciations (adjective_id, pronunciation) VALUES (?, ?)"; | ||
641 | |||
642 | break; | ||
643 | } | ||
644 | |||
645 | case 4: // Adverb | ||
646 | { | ||
647 | query = "INSERT INTO adverb_pronunciations (adverb_id, pronunciation) VALUES (?, ?)"; | ||
648 | |||
649 | break; | ||
650 | } | ||
651 | } | ||
652 | |||
653 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
654 | { | ||
655 | db_error(ppdb, query); | ||
656 | } | ||
657 | |||
658 | sqlite3_bind_int(ppstmt, 1, rowid); | ||
659 | sqlite3_bind_text(ppstmt, 2, pronunciation.c_str(), pronunciation.length(), SQLITE_STATIC); | ||
660 | |||
661 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
662 | { | ||
663 | db_error(ppdb, query); | ||
664 | } | ||
665 | |||
666 | sqlite3_finalize(ppstmt); | ||
667 | } | ||
668 | } | ||
669 | } | ||
670 | } | ||
671 | |||
672 | // While we're working on s | ||
673 | { | ||
674 | progress ppgs("Writing word synonyms...", wn.size()); | ||
675 | for (auto sense : wn) | ||
676 | { | ||
677 | ppgs.update(); | ||
678 | |||
679 | for (auto word1 : sense.second) | ||
680 | { | ||
681 | for (auto word2 : sense.second) | ||
682 | { | ||
683 | if (word1 != word2) | ||
684 | { | ||
685 | std::string query; | ||
686 | switch (sense.first / 100000000) | ||
687 | { | ||
688 | case 1: // Noun | ||
689 | { | ||
690 | query = "INSERT INTO noun_synonymy (noun_1_id, noun_2_id) VALUES (?, ?)"; | ||
691 | |||
692 | break; | ||
693 | } | ||
694 | |||
695 | case 2: // Verb | ||
696 | { | ||
697 | // Ignore | ||
698 | |||
699 | break; | ||
700 | } | ||
701 | |||
702 | case 3: // Adjective | ||
703 | { | ||
704 | query = "INSERT INTO adjective_synonymy (adjective_1_id, adjective_2_id) VALUES (?, ?)"; | ||
705 | |||
706 | break; | ||
707 | } | ||
708 | |||
709 | case 4: // Adverb | ||
710 | { | ||
711 | query = "INSERT INTO adverb_synonymy (adverb_1_id, adverb_2_id) VALUES (?, ?)"; | ||
712 | |||
713 | break; | ||
714 | } | ||
715 | } | ||
716 | |||
717 | sqlite3_stmt* ppstmt; | ||
718 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
719 | { | ||
720 | db_error(ppdb, query); | ||
721 | } | ||
722 | |||
723 | sqlite3_bind_int(ppstmt, 1, word1.second); | ||
724 | sqlite3_bind_int(ppstmt, 2, word2.second); | ||
725 | |||
726 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
727 | { | ||
728 | db_error(ppdb, query); | ||
729 | } | ||
730 | |||
731 | sqlite3_finalize(ppstmt); | ||
732 | } | ||
733 | } | ||
734 | } | ||
735 | } | ||
736 | } | ||
737 | |||
738 | // ant table | ||
739 | { | ||
740 | std::ifstream wnantfile(wnpref + "wn_ant.pl"); | ||
741 | if (!wnantfile.is_open()) | ||
742 | { | ||
743 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
744 | print_usage(); | ||
745 | } | ||
746 | |||
747 | std::list<std::string> lines; | ||
748 | for (;;) | ||
749 | { | ||
750 | std::string line; | ||
751 | if (!getline(wnantfile, line)) | ||
752 | { | ||
753 | break; | ||
754 | } | ||
755 | |||
756 | if (line.back() == '\r') | ||
757 | { | ||
758 | line.pop_back(); | ||
759 | } | ||
760 | |||
761 | lines.push_back(line); | ||
762 | } | ||
763 | |||
764 | progress ppgs("Writing antonyms...", lines.size()); | ||
765 | for (auto line : lines) | ||
766 | { | ||
767 | ppgs.update(); | ||
768 | |||
769 | std::regex relation("^ant\\(([134]\\d{8}),(\\d+),([134]\\d{8}),(\\d+)\\)\\."); | ||
770 | std::smatch relation_data; | ||
771 | if (!std::regex_search(line, relation_data, relation)) | ||
772 | { | ||
773 | continue; | ||
774 | } | ||
775 | |||
776 | int synset_id_1 = stoi(relation_data[1]); | ||
777 | int wnum_1 = stoi(relation_data[2]); | ||
778 | int synset_id_2 = stoi(relation_data[3]); | ||
779 | int wnum_2 = stoi(relation_data[4]); | ||
780 | |||
781 | std::string query; | ||
782 | switch (synset_id_1 / 100000000) | ||
783 | { | ||
784 | case 1: // Noun | ||
785 | { | ||
786 | query = "INSERT INTO noun_antonymy (noun_1_id, noun_2_id) VALUES (?, ?)"; | ||
787 | |||
788 | break; | ||
789 | } | ||
790 | |||
791 | case 2: // Verb | ||
792 | { | ||
793 | // Ignore | ||
794 | |||
795 | break; | ||
796 | } | ||
797 | |||
798 | case 3: // Adjective | ||
799 | { | ||
800 | query = "INSERT INTO adjective_antonymy (adjective_1_id, adjective_2_id) VALUES (?, ?)"; | ||
801 | |||
802 | break; | ||
803 | } | ||
804 | |||
805 | case 4: // Adverb | ||
806 | { | ||
807 | query = "INSERT INTO adverb_antonymy (adverb_1_id, adverb_2_id) VALUES (?, ?)"; | ||
808 | |||
809 | break; | ||
810 | } | ||
811 | } | ||
812 | |||
813 | sqlite3_stmt* ppstmt; | ||
814 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
815 | { | ||
816 | db_error(ppdb, query); | ||
817 | } | ||
818 | |||
819 | sqlite3_bind_int(ppstmt, 1, wn[synset_id_1][wnum_1]); | ||
820 | sqlite3_bind_int(ppstmt, 2, wn[synset_id_2][wnum_2]); | ||
821 | |||
822 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
823 | { | ||
824 | db_error(ppdb, query); | ||
825 | } | ||
826 | |||
827 | sqlite3_finalize(ppstmt); | ||
828 | } | ||
829 | } | ||
830 | |||
831 | // at table | ||
832 | { | ||
833 | std::ifstream wnatfile(wnpref + "wn_at.pl"); | ||
834 | if (!wnatfile.is_open()) | ||
835 | { | ||
836 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
837 | print_usage(); | ||
838 | } | ||
839 | |||
840 | std::list<std::string> lines; | ||
841 | for (;;) | ||
842 | { | ||
843 | std::string line; | ||
844 | if (!getline(wnatfile, line)) | ||
845 | { | ||
846 | break; | ||
847 | } | ||
848 | |||
849 | if (line.back() == '\r') | ||
850 | { | ||
851 | line.pop_back(); | ||
852 | } | ||
853 | |||
854 | lines.push_back(line); | ||
855 | } | ||
856 | |||
857 | progress ppgs("Writing variations...", lines.size()); | ||
858 | for (auto line : lines) | ||
859 | { | ||
860 | ppgs.update(); | ||
861 | |||
862 | std::regex relation("^at\\((1\\d{8}),(3\\d{8})\\)\\."); | ||
863 | std::smatch relation_data; | ||
864 | if (!std::regex_search(line, relation_data, relation)) | ||
865 | { | ||
866 | continue; | ||
867 | } | ||
868 | |||
869 | int synset_id_1 = stoi(relation_data[1]); | ||
870 | int synset_id_2 = stoi(relation_data[2]); | ||
871 | std::string query("INSERT INTO variation (noun_id, adjective_id) VALUES (?, ?)"); | ||
872 | |||
873 | for (auto mapping1 : wn[synset_id_1]) | ||
874 | { | ||
875 | for (auto mapping2 : wn[synset_id_2]) | ||
876 | { | ||
877 | sqlite3_stmt* ppstmt; | ||
878 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
879 | { | ||
880 | db_error(ppdb, query); | ||
881 | } | ||
882 | |||
883 | sqlite3_bind_int(ppstmt, 1, mapping1.second); | ||
884 | sqlite3_bind_int(ppstmt, 2, mapping2.second); | ||
885 | |||
886 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
887 | { | ||
888 | db_error(ppdb, query); | ||
889 | } | ||
890 | |||
891 | sqlite3_finalize(ppstmt); | ||
892 | } | ||
893 | } | ||
894 | } | ||
895 | } | ||
896 | |||
897 | // der table | ||
898 | { | ||
899 | std::ifstream wnderfile(wnpref + "wn_der.pl"); | ||
900 | if (!wnderfile.is_open()) | ||
901 | { | ||
902 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
903 | print_usage(); | ||
904 | } | ||
905 | |||
906 | std::list<std::string> lines; | ||
907 | for (;;) | ||
908 | { | ||
909 | std::string line; | ||
910 | if (!getline(wnderfile, line)) | ||
911 | { | ||
912 | break; | ||
913 | } | ||
914 | |||
915 | if (line.back() == '\r') | ||
916 | { | ||
917 | line.pop_back(); | ||
918 | } | ||
919 | |||
920 | lines.push_back(line); | ||
921 | } | ||
922 | |||
923 | progress ppgs("Writing morphological derivation...", lines.size()); | ||
924 | for (auto line : lines) | ||
925 | { | ||
926 | ppgs.update(); | ||
927 | |||
928 | std::regex relation("^der\\(([134]\\d{8}),(\\d+),([134]\\d{8}),(\\d+)\\)\\."); | ||
929 | std::smatch relation_data; | ||
930 | if (!std::regex_search(line, relation_data, relation)) | ||
931 | { | ||
932 | continue; | ||
933 | } | ||
934 | |||
935 | int synset_id_1 = stoi(relation_data[1]); | ||
936 | int wnum_1 = stoi(relation_data[2]); | ||
937 | int synset_id_2 = stoi(relation_data[3]); | ||
938 | int wnum_2 = stoi(relation_data[4]); | ||
939 | std::string query; | ||
940 | switch (synset_id_1 / 100000000) | ||
941 | { | ||
942 | case 1: // Noun | ||
943 | { | ||
944 | switch (synset_id_2 / 100000000) | ||
945 | { | ||
946 | case 1: // Noun | ||
947 | { | ||
948 | query = "INSERT INTO noun_noun_derivation (noun_1_id, noun_2_id) VALUES (?, ?)"; | ||
949 | break; | ||
950 | } | ||
951 | |||
952 | case 3: // Adjective | ||
953 | { | ||
954 | query = "INSERT INTO noun_adjective_derivation (noun_id, adjective_id) VALUES (?, ?)"; | ||
955 | break; | ||
956 | } | ||
957 | |||
958 | case 4: // Adverb | ||
959 | { | ||
960 | query = "INSERT INTO noun_adverb_derivation (noun_id, adverb_id) VALUES (?, ?)"; | ||
961 | break; | ||
962 | } | ||
963 | } | ||
964 | |||
965 | break; | ||
966 | } | ||
967 | |||
968 | case 3: // Adjective | ||
969 | { | ||
970 | switch (synset_id_2 / 100000000) | ||
971 | { | ||
972 | case 1: // Noun | ||
973 | { | ||
974 | query = "INSERT INTO noun_adjective_derivation (adjective_id, noun_id) VALUES (?, ?)"; | ||
975 | break; | ||
976 | } | ||
977 | |||
978 | case 3: // Adjective | ||
979 | { | ||
980 | query = "INSERT INTO adjective_adjective_derivation (adjective_id, adjective_id) VALUES (?, ?)"; | ||
981 | break; | ||
982 | } | ||
983 | |||
984 | case 4: // Adverb | ||
985 | { | ||
986 | query = "INSERT INTO adjective_adverb_derivation (adjective_id, adverb_id) VALUES (?, ?)"; | ||
987 | break; | ||
988 | } | ||
989 | } | ||
990 | |||
991 | break; | ||
992 | } | ||
993 | |||
994 | case 4: // Adverb | ||
995 | { | ||
996 | switch (synset_id_2 / 100000000) | ||
997 | { | ||
998 | case 1: // Noun | ||
999 | { | ||
1000 | query = "INSERT INTO noun_adverb_derivation (adverb_id, noun_id) VALUES (?, ?)"; | ||
1001 | break; | ||
1002 | } | ||
1003 | |||
1004 | case 3: // Adjective | ||
1005 | { | ||
1006 | query = "INSERT INTO adjective_adverb_derivation (adverb_id, adjective_id) VALUES (?, ?)"; | ||
1007 | break; | ||
1008 | } | ||
1009 | |||
1010 | case 4: // Adverb | ||
1011 | { | ||
1012 | query = "INSERT INTO adverb_adverb_derivation (adverb_1_id, adverb_2_id) VALUES (?, ?)"; | ||
1013 | break; | ||
1014 | } | ||
1015 | } | ||
1016 | |||
1017 | break; | ||
1018 | } | ||
1019 | } | ||
1020 | |||
1021 | sqlite3_stmt* ppstmt; | ||
1022 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
1023 | { | ||
1024 | db_error(ppdb, query); | ||
1025 | } | ||
1026 | |||
1027 | sqlite3_bind_int(ppstmt, 1, wn[synset_id_1][wnum_1]); | ||
1028 | sqlite3_bind_int(ppstmt, 2, wn[synset_id_2][wnum_2]); | ||
1029 | |||
1030 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1031 | { | ||
1032 | db_error(ppdb, query); | ||
1033 | } | ||
1034 | |||
1035 | sqlite3_finalize(ppstmt); | ||
1036 | } | ||
1037 | } | ||
1038 | |||
1039 | // hyp table | ||
1040 | { | ||
1041 | std::ifstream wnhypfile(wnpref + "wn_hyp.pl"); | ||
1042 | if (!wnhypfile.is_open()) | ||
1043 | { | ||
1044 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1045 | print_usage(); | ||
1046 | } | ||
1047 | |||
1048 | std::list<std::string> lines; | ||
1049 | for (;;) | ||
1050 | { | ||
1051 | std::string line; | ||
1052 | if (!getline(wnhypfile, line)) | ||
1053 | { | ||
1054 | break; | ||
1055 | } | ||
1056 | |||
1057 | if (line.back() == '\r') | ||
1058 | { | ||
1059 | line.pop_back(); | ||
1060 | } | ||
1061 | |||
1062 | lines.push_back(line); | ||
1063 | } | ||
1064 | |||
1065 | progress ppgs("Writing hypernyms...", lines.size()); | ||
1066 | for (auto line : lines) | ||
1067 | { | ||
1068 | ppgs.update(); | ||
1069 | |||
1070 | std::regex relation("^hyp\\((1\\d{8}),(1\\d{8})\\)\\."); | ||
1071 | std::smatch relation_data; | ||
1072 | if (!std::regex_search(line, relation_data, relation)) | ||
1073 | { | ||
1074 | continue; | ||
1075 | } | ||
1076 | |||
1077 | int synset_id_1 = stoi(relation_data[1]); | ||
1078 | int synset_id_2 = stoi(relation_data[2]); | ||
1079 | std::string query("INSERT INTO hypernymy (hyponym_id, hypernym_id) VALUES (?, ?)"); | ||
1080 | |||
1081 | for (auto mapping1 : wn[synset_id_1]) | ||
1082 | { | ||
1083 | for (auto mapping2 : wn[synset_id_2]) | ||
1084 | { | ||
1085 | sqlite3_stmt* ppstmt; | ||
1086 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
1087 | { | ||
1088 | db_error(ppdb, query); | ||
1089 | } | ||
1090 | |||
1091 | sqlite3_bind_int(ppstmt, 1, mapping1.second); | ||
1092 | sqlite3_bind_int(ppstmt, 2, mapping2.second); | ||
1093 | |||
1094 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1095 | { | ||
1096 | db_error(ppdb, query); | ||
1097 | } | ||
1098 | |||
1099 | sqlite3_finalize(ppstmt); | ||
1100 | } | ||
1101 | } | ||
1102 | } | ||
1103 | } | ||
1104 | |||
1105 | // ins table | ||
1106 | { | ||
1107 | std::ifstream wninsfile(wnpref + "wn_ins.pl"); | ||
1108 | if (!wninsfile.is_open()) | ||
1109 | { | ||
1110 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1111 | print_usage(); | ||
1112 | } | ||
1113 | |||
1114 | std::list<std::string> lines; | ||
1115 | for (;;) | ||
1116 | { | ||
1117 | std::string line; | ||
1118 | if (!getline(wninsfile, line)) | ||
1119 | { | ||
1120 | break; | ||
1121 | } | ||
1122 | |||
1123 | if (line.back() == '\r') | ||
1124 | { | ||
1125 | line.pop_back(); | ||
1126 | } | ||
1127 | |||
1128 | lines.push_back(line); | ||
1129 | } | ||
1130 | |||
1131 | progress ppgs("Writing instantiations...", lines.size()); | ||
1132 | for (auto line : lines) | ||
1133 | { | ||
1134 | ppgs.update(); | ||
1135 | |||
1136 | std::regex relation("^ins\\((1\\d{8}),(1\\d{8})\\)\\."); | ||
1137 | std::smatch relation_data; | ||
1138 | if (!std::regex_search(line, relation_data, relation)) | ||
1139 | { | ||
1140 | continue; | ||
1141 | } | ||
1142 | |||
1143 | int synset_id_1 = stoi(relation_data[1]); | ||
1144 | int synset_id_2 = stoi(relation_data[2]); | ||
1145 | std::string query("INSERT INTO instantiation (instance_id, class_id) VALUES (?, ?)"); | ||
1146 | |||
1147 | for (auto mapping1 : wn[synset_id_1]) | ||
1148 | { | ||
1149 | for (auto mapping2 : wn[synset_id_2]) | ||
1150 | { | ||
1151 | sqlite3_stmt* ppstmt; | ||
1152 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
1153 | { | ||
1154 | db_error(ppdb, query); | ||
1155 | } | ||
1156 | |||
1157 | sqlite3_bind_int(ppstmt, 1, mapping1.second); | ||
1158 | sqlite3_bind_int(ppstmt, 2, mapping2.second); | ||
1159 | |||
1160 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1161 | { | ||
1162 | db_error(ppdb, query); | ||
1163 | } | ||
1164 | |||
1165 | sqlite3_finalize(ppstmt); | ||
1166 | } | ||
1167 | } | ||
1168 | } | ||
1169 | } | ||
1170 | |||
1171 | // mm table | ||
1172 | { | ||
1173 | std::ifstream wnmmfile(wnpref + "wn_mm.pl"); | ||
1174 | if (!wnmmfile.is_open()) | ||
1175 | { | ||
1176 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1177 | print_usage(); | ||
1178 | } | ||
1179 | |||
1180 | std::list<std::string> lines; | ||
1181 | for (;;) | ||
1182 | { | ||
1183 | std::string line; | ||
1184 | if (!getline(wnmmfile, line)) | ||
1185 | { | ||
1186 | break; | ||
1187 | } | ||
1188 | |||
1189 | if (line.back() == '\r') | ||
1190 | { | ||
1191 | line.pop_back(); | ||
1192 | } | ||
1193 | |||
1194 | lines.push_back(line); | ||
1195 | } | ||
1196 | |||
1197 | progress ppgs("Writing member meronyms...", lines.size()); | ||
1198 | for (auto line : lines) | ||
1199 | { | ||
1200 | ppgs.update(); | ||
1201 | |||
1202 | std::regex relation("^mm\\((1\\d{8}),(1\\d{8})\\)\\."); | ||
1203 | std::smatch relation_data; | ||
1204 | if (!std::regex_search(line, relation_data, relation)) | ||
1205 | { | ||
1206 | continue; | ||
1207 | } | ||
1208 | |||
1209 | int synset_id_1 = stoi(relation_data[1]); | ||
1210 | int synset_id_2 = stoi(relation_data[2]); | ||
1211 | std::string query("INSERT INTO member_meronymy (holonym_id, meronym_id) VALUES (?, ?)"); | ||
1212 | |||
1213 | for (auto mapping1 : wn[synset_id_1]) | ||
1214 | { | ||
1215 | for (auto mapping2 : wn[synset_id_2]) | ||
1216 | { | ||
1217 | sqlite3_stmt* ppstmt; | ||
1218 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
1219 | { | ||
1220 | db_error(ppdb, query); | ||
1221 | } | ||
1222 | |||
1223 | sqlite3_bind_int(ppstmt, 1, mapping1.second); | ||
1224 | sqlite3_bind_int(ppstmt, 2, mapping2.second); | ||
1225 | |||
1226 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1227 | { | ||
1228 | db_error(ppdb, query); | ||
1229 | } | ||
1230 | |||
1231 | sqlite3_finalize(ppstmt); | ||
1232 | } | ||
1233 | } | ||
1234 | } | ||
1235 | } | ||
1236 | |||
1237 | // ms table | ||
1238 | { | ||
1239 | std::ifstream wnmsfile(wnpref + "wn_ms.pl"); | ||
1240 | if (!wnmsfile.is_open()) | ||
1241 | { | ||
1242 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1243 | print_usage(); | ||
1244 | } | ||
1245 | |||
1246 | std::list<std::string> lines; | ||
1247 | for (;;) | ||
1248 | { | ||
1249 | std::string line; | ||
1250 | if (!getline(wnmsfile, line)) | ||
1251 | { | ||
1252 | break; | ||
1253 | } | ||
1254 | |||
1255 | if (line.back() == '\r') | ||
1256 | { | ||
1257 | line.pop_back(); | ||
1258 | } | ||
1259 | |||
1260 | lines.push_back(line); | ||
1261 | } | ||
1262 | |||
1263 | progress ppgs("Writing substance meronyms...", lines.size()); | ||
1264 | for (auto line : lines) | ||
1265 | { | ||
1266 | ppgs.update(); | ||
1267 | |||
1268 | std::regex relation("^ms\\((1\\d{8}),(1\\d{8})\\)\\."); | ||
1269 | std::smatch relation_data; | ||
1270 | if (!std::regex_search(line, relation_data, relation)) | ||
1271 | { | ||
1272 | continue; | ||
1273 | } | ||
1274 | |||
1275 | int synset_id_1 = stoi(relation_data[1]); | ||
1276 | int synset_id_2 = stoi(relation_data[2]); | ||
1277 | std::string query("INSERT INTO substance_meronymy (holonym_id, meronym_id) VALUES (?, ?)"); | ||
1278 | |||
1279 | for (auto mapping1 : wn[synset_id_1]) | ||
1280 | { | ||
1281 | for (auto mapping2 : wn[synset_id_2]) | ||
1282 | { | ||
1283 | sqlite3_stmt* ppstmt; | ||
1284 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
1285 | { | ||
1286 | db_error(ppdb, query); | ||
1287 | } | ||
1288 | |||
1289 | sqlite3_bind_int(ppstmt, 1, mapping1.second); | ||
1290 | sqlite3_bind_int(ppstmt, 2, mapping2.second); | ||
1291 | |||
1292 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1293 | { | ||
1294 | db_error(ppdb, query); | ||
1295 | } | ||
1296 | |||
1297 | sqlite3_finalize(ppstmt); | ||
1298 | } | ||
1299 | } | ||
1300 | } | ||
1301 | } | ||
1302 | |||
1303 | // mm table | ||
1304 | { | ||
1305 | std::ifstream wnmpfile(wnpref + "wn_mp.pl"); | ||
1306 | if (!wnmpfile.is_open()) | ||
1307 | { | ||
1308 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1309 | print_usage(); | ||
1310 | } | ||
1311 | |||
1312 | std::list<std::string> lines; | ||
1313 | for (;;) | ||
1314 | { | ||
1315 | std::string line; | ||
1316 | if (!getline(wnmpfile, line)) | ||
1317 | { | ||
1318 | break; | ||
1319 | } | ||
1320 | |||
1321 | if (line.back() == '\r') | ||
1322 | { | ||
1323 | line.pop_back(); | ||
1324 | } | ||
1325 | |||
1326 | lines.push_back(line); | ||
1327 | } | ||
1328 | |||
1329 | progress ppgs("Writing part meronyms...", lines.size()); | ||
1330 | for (auto line : lines) | ||
1331 | { | ||
1332 | ppgs.update(); | ||
1333 | |||
1334 | std::regex relation("^mp\\((1\\d{8}),(1\\d{8})\\)\\."); | ||
1335 | std::smatch relation_data; | ||
1336 | if (!std::regex_search(line, relation_data, relation)) | ||
1337 | { | ||
1338 | continue; | ||
1339 | } | ||
1340 | |||
1341 | int synset_id_1 = stoi(relation_data[1]); | ||
1342 | int synset_id_2 = stoi(relation_data[2]); | ||
1343 | std::string query("INSERT INTO part_meronymy (holonym_id, meronym_id) VALUES (?, ?)"); | ||
1344 | |||
1345 | for (auto mapping1 : wn[synset_id_1]) | ||
1346 | { | ||
1347 | for (auto mapping2 : wn[synset_id_2]) | ||
1348 | { | ||
1349 | sqlite3_stmt* ppstmt; | ||
1350 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
1351 | { | ||
1352 | db_error(ppdb, query); | ||
1353 | } | ||
1354 | |||
1355 | sqlite3_bind_int(ppstmt, 1, mapping1.second); | ||
1356 | sqlite3_bind_int(ppstmt, 2, mapping2.second); | ||
1357 | |||
1358 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1359 | { | ||
1360 | db_error(ppdb, query); | ||
1361 | } | ||
1362 | |||
1363 | sqlite3_finalize(ppstmt); | ||
1364 | } | ||
1365 | } | ||
1366 | } | ||
1367 | } | ||
1368 | |||
1369 | // per table | ||
1370 | { | ||
1371 | std::ifstream wnperfile(wnpref + "wn_per.pl"); | ||
1372 | if (!wnperfile.is_open()) | ||
1373 | { | ||
1374 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1375 | print_usage(); | ||
1376 | } | ||
1377 | |||
1378 | std::list<std::string> lines; | ||
1379 | for (;;) | ||
1380 | { | ||
1381 | std::string line; | ||
1382 | if (!getline(wnperfile, line)) | ||
1383 | { | ||
1384 | break; | ||
1385 | } | ||
1386 | |||
1387 | if (line.back() == '\r') | ||
1388 | { | ||
1389 | line.pop_back(); | ||
1390 | } | ||
1391 | |||
1392 | lines.push_back(line); | ||
1393 | } | ||
1394 | |||
1395 | progress ppgs("Writing pertainyms and mannernyms...", lines.size()); | ||
1396 | for (auto line : lines) | ||
1397 | { | ||
1398 | ppgs.update(); | ||
1399 | |||
1400 | std::regex relation("^per\\(([34]\\d{8}),(\\d+),([13]\\d{8}),(\\d+)\\)\\."); | ||
1401 | std::smatch relation_data; | ||
1402 | if (!std::regex_search(line, relation_data, relation)) | ||
1403 | { | ||
1404 | continue; | ||
1405 | } | ||
1406 | |||
1407 | int synset_id_1 = stoi(relation_data[1]); | ||
1408 | int wnum_1 = stoi(relation_data[2]); | ||
1409 | int synset_id_2 = stoi(relation_data[3]); | ||
1410 | int wnum_2 = stoi(relation_data[4]); | ||
1411 | std::string query; | ||
1412 | switch (synset_id_1 / 100000000) | ||
1413 | { | ||
1414 | case 3: // Adjective | ||
1415 | { | ||
1416 | // This is a pertainym, the second word should be a noun | ||
1417 | // Technically it can be an adjective but we're ignoring that | ||
1418 | if (synset_id_2 / 100000000 != 1) | ||
1419 | { | ||
1420 | continue; | ||
1421 | } | ||
1422 | |||
1423 | query = "INSERT INTO pertainymy (pertainym_id, noun_id) VALUES (?, ?)"; | ||
1424 | |||
1425 | break; | ||
1426 | } | ||
1427 | |||
1428 | case 4: // Adverb | ||
1429 | { | ||
1430 | // This is a mannernym, the second word should be an adjective | ||
1431 | if (synset_id_2 / 100000000 != 3) | ||
1432 | { | ||
1433 | continue; | ||
1434 | } | ||
1435 | |||
1436 | query = "INSERT INTO mannernymy (mannernym_id, adjective_id) VALUES (?, ?)"; | ||
1437 | |||
1438 | break; | ||
1439 | } | ||
1440 | } | ||
1441 | |||
1442 | sqlite3_stmt* ppstmt; | ||
1443 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
1444 | { | ||
1445 | db_error(ppdb, query); | ||
1446 | } | ||
1447 | |||
1448 | sqlite3_bind_int(ppstmt, 1, wn[synset_id_1][wnum_1]); | ||
1449 | sqlite3_bind_int(ppstmt, 2, wn[synset_id_2][wnum_2]); | ||
1450 | |||
1451 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1452 | { | ||
1453 | db_error(ppdb, query); | ||
1454 | } | ||
1455 | |||
1456 | sqlite3_finalize(ppstmt); | ||
1457 | } | ||
1458 | } | ||
1459 | |||
1460 | // sa table | ||
1461 | { | ||
1462 | std::ifstream wnsafile(wnpref + "wn_sa.pl"); | ||
1463 | if (!wnsafile.is_open()) | ||
1464 | { | ||
1465 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1466 | print_usage(); | ||
1467 | } | ||
1468 | |||
1469 | std::list<std::string> lines; | ||
1470 | for (;;) | ||
1471 | { | ||
1472 | std::string line; | ||
1473 | if (!getline(wnsafile, line)) | ||
1474 | { | ||
1475 | break; | ||
1476 | } | ||
1477 | |||
1478 | if (line.back() == '\r') | ||
1479 | { | ||
1480 | line.pop_back(); | ||
1481 | } | ||
1482 | |||
1483 | lines.push_back(line); | ||
1484 | } | ||
1485 | |||
1486 | progress ppgs("Writing specifications...", lines.size()); | ||
1487 | for (auto line : lines) | ||
1488 | { | ||
1489 | ppgs.update(); | ||
1490 | |||
1491 | std::regex relation("^per\\((3\\d{8}),(\\d+),(3\\d{8}),(\\d+)\\)\\."); | ||
1492 | std::smatch relation_data; | ||
1493 | if (!std::regex_search(line, relation_data, relation)) | ||
1494 | { | ||
1495 | continue; | ||
1496 | } | ||
1497 | |||
1498 | int synset_id_1 = stoi(relation_data[1]); | ||
1499 | int wnum_1 = stoi(relation_data[2]); | ||
1500 | int synset_id_2 = stoi(relation_data[3]); | ||
1501 | int wnum_2 = stoi(relation_data[4]); | ||
1502 | std::string query("INSERT INTO specification (general_id, specific_id) VALUES (?, ?)"); | ||
1503 | |||
1504 | sqlite3_stmt* ppstmt; | ||
1505 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.length(), &ppstmt, NULL) != SQLITE_OK) | ||
1506 | { | ||
1507 | db_error(ppdb, query); | ||
1508 | } | ||
1509 | |||
1510 | sqlite3_bind_int(ppstmt, 1, wn[synset_id_1][wnum_1]); | ||
1511 | sqlite3_bind_int(ppstmt, 2, wn[synset_id_2][wnum_2]); | ||
1512 | |||
1513 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1514 | { | ||
1515 | db_error(ppdb, query); | ||
1516 | } | ||
1517 | |||
1518 | sqlite3_finalize(ppstmt); | ||
1519 | } | ||
1520 | } | ||
1521 | |||
1522 | // sim table | ||
1523 | { | ||
1524 | std::ifstream wnsimfile(wnpref + "wn_sim.pl"); | ||
1525 | if (!wnsimfile.is_open()) | ||
1526 | { | ||
1527 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1528 | print_usage(); | ||
1529 | } | ||
1530 | |||
1531 | std::list<std::string> lines; | ||
1532 | for (;;) | ||
1533 | { | ||
1534 | std::string line; | ||
1535 | if (!getline(wnsimfile, line)) | ||
1536 | { | ||
1537 | break; | ||
1538 | } | ||
1539 | |||
1540 | if (line.back() == '\r') | ||
1541 | { | ||
1542 | line.pop_back(); | ||
1543 | } | ||
1544 | |||
1545 | lines.push_back(line); | ||
1546 | } | ||
1547 | |||
1548 | progress ppgs("Writing sense synonyms...", lines.size()); | ||
1549 | for (auto line : lines) | ||
1550 | { | ||
1551 | ppgs.update(); | ||
1552 | |||
1553 | std::regex relation("^sim\\((3\\d{8}),(3\\d{8})\\)\\."); | ||
1554 | std::smatch relation_data; | ||
1555 | if (!std::regex_search(line, relation_data, relation)) | ||
1556 | { | ||
1557 | continue; | ||
1558 | } | ||
1559 | |||
1560 | int synset_id_1 = stoi(relation_data[1]); | ||
1561 | int synset_id_2 = stoi(relation_data[2]); | ||
1562 | std::string query("INSERT INTO adjective_synonymy (adjective_1_id, adjective_2_id) VALUES (?, ?)"); | ||
1563 | |||
1564 | for (auto mapping1 : wn[synset_id_1]) | ||
1565 | { | ||
1566 | for (auto mapping2 : wn[synset_id_2]) | ||
1567 | { | ||
1568 | sqlite3_stmt* ppstmt; | ||
1569 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
1570 | { | ||
1571 | db_error(ppdb, query); | ||
1572 | } | ||
1573 | |||
1574 | sqlite3_bind_int(ppstmt, 1, mapping1.second); | ||
1575 | sqlite3_bind_int(ppstmt, 2, mapping2.second); | ||
1576 | |||
1577 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1578 | { | ||
1579 | db_error(ppdb, query); | ||
1580 | } | ||
1581 | |||
1582 | sqlite3_reset(ppstmt); | ||
1583 | sqlite3_clear_bindings(ppstmt); | ||
1584 | |||
1585 | sqlite3_bind_int(ppstmt, 1, mapping2.second); | ||
1586 | sqlite3_bind_int(ppstmt, 2, mapping1.second); | ||
1587 | |||
1588 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1589 | { | ||
1590 | db_error(ppdb, query); | ||
1591 | } | ||
1592 | |||
1593 | sqlite3_finalize(ppstmt); | ||
1594 | } | ||
1595 | } | ||
1596 | } | ||
1597 | } | ||
1598 | |||
1599 | // syntax table | ||
1600 | { | ||
1601 | std::ifstream wnsyntaxfile(wnpref + "wn_syntax.pl"); | ||
1602 | if (!wnsyntaxfile.is_open()) | ||
1603 | { | ||
1604 | std::cout << "Invalid WordNet data directory." << std::endl; | ||
1605 | print_usage(); | ||
1606 | } | ||
1607 | |||
1608 | std::list<std::string> lines; | ||
1609 | for (;;) | ||
1610 | { | ||
1611 | std::string line; | ||
1612 | if (!getline(wnsyntaxfile, line)) | ||
1613 | { | ||
1614 | break; | ||
1615 | } | ||
1616 | |||
1617 | if (line.back() == '\r') | ||
1618 | { | ||
1619 | line.pop_back(); | ||
1620 | } | ||
1621 | |||
1622 | lines.push_back(line); | ||
1623 | } | ||
1624 | |||
1625 | progress ppgs("Writing adjective syntax markers...", lines.size()); | ||
1626 | for (auto line : lines) | ||
1627 | { | ||
1628 | ppgs.update(); | ||
1629 | |||
1630 | std::regex relation("^syntax\\((3\\d{8}),(\\d+),([ipa])p?\\)\\."); | ||
1631 | std::smatch relation_data; | ||
1632 | if (!std::regex_search(line, relation_data, relation)) | ||
1633 | { | ||
1634 | continue; | ||
1635 | } | ||
1636 | |||
1637 | int synset_id = stoi(relation_data[1]); | ||
1638 | int wnum = stoi(relation_data[2]); | ||
1639 | std::string syn = relation_data[3]; | ||
1640 | std::string query("UPDATE adjectives SET position = ? WHERE adjective_id = ?"); | ||
1641 | |||
1642 | sqlite3_stmt* ppstmt; | ||
1643 | if (sqlite3_prepare_v2(ppdb, query.c_str(), query.size(), &ppstmt, NULL) != SQLITE_OK) | ||
1644 | { | ||
1645 | db_error(ppdb, query); | ||
1646 | } | ||
1647 | |||
1648 | sqlite3_bind_text(ppstmt, 1, syn.c_str(), 1, SQLITE_STATIC); | ||
1649 | sqlite3_bind_int(ppstmt, 2, wn[synset_id][wnum]); | ||
1650 | |||
1651 | if (sqlite3_step(ppstmt) != SQLITE_DONE) | ||
1652 | { | ||
1653 | db_error(ppdb, query); | ||
1654 | } | ||
1655 | |||
1656 | sqlite3_finalize(ppstmt); | ||
1657 | } | ||
1658 | } | ||
1659 | |||
1660 | sqlite3_close_v2(ppdb); | ||
1661 | |||
1662 | std::cout << "Done." << std::endl; | ||
1663 | } | ||