Added emoji freevar

Strings of emojis are tokenized separately from anything else, and added to an emoticon freevar, which is mixed in with regular emoticons like :P. This breaks old-style freevars like $name$ and $noun$ so some legacy support for compatibility is left in but eventually $name$ should be made into an actual new freevar. Emoji data is from gemoji (https://github.com/github/gemoji).

13 files changed, 1064 insertions, 65 deletions

@@ -8,7 +8,7 @@ find_package(curl)
 if (YamlCpp_FOUND AND CURL_FOUND)
   add_subdirectory(vendor/twitcurl/libtwitcurl)
   include_directories(vendor/twitcurl/libtwitcurl)
-  add_executable(rawr-ebooks ebooks.cpp kgramstats.cpp freevars.cpp histogram.cpp)
+  add_executable(rawr-ebooks ebooks.cpp kgramstats.cpp freevars.cpp histogram.cpp prefix_search.cpp)
   set_property(TARGET rawr-ebooks PROPERTY CXX_STANDARD 11)
   set_property(TARGET rawr-ebooks PROPERTY CXX_STANDARD_REQUIRED ON)
   target_link_libraries(rawr-ebooks ${YamlCpp_LIBRARIES} twitcurl ${CURL_LIBRARIES})
@@ -16,6 +16,6 @@ else (YamlCpp_FOUND AND CURL_FOUND)
   message(STATUS "rawr-ebooks requires yaml-cpp and twitcurl; without these, we will only make rawr-gen")
 endif (YamlCpp_FOUND AND CURL_FOUND)
 
-add_executable(rawr-gen gen.cpp kgramstats.cpp freevars.cpp histogram.cpp)
+add_executable(rawr-gen gen.cpp kgramstats.cpp freevars.cpp histogram.cpp prefix_search.cpp)
 set_property(TARGET rawr-gen PROPERTY CXX_STANDARD 11)
 set_property(TARGET rawr-gen PROPERTY CXX_STANDARD_REQUIRED ON)
@@ -10,7 +10,6 @@
 #include <twitcurl.h>
 #include <unistd.h>
 #include <yaml-cpp/yaml.h>
-#include "freevars.h"
 
 int main(int argc, char** args)
 {
@@ -29,11 +28,6 @@ int main(int argc, char** args)
 
   std::cout << "Preprocessing corpus..." << std::endl;
   kgramstats* stats = new kgramstats(corpus, 4);
-    
-  std::cout << "Preprocessing freevars..." << std::endl;
-  freevars* vars = new freevars();
-  vars->addVar("name", "names.txt");
-  vars->addVar("noun", "nouns.txt");
   
   twitCurl twitter;
   twitter.getOAuth().setConsumerKey(config["consumer_key"].as<std::string>());
@@ -45,7 +39,7 @@ int main(int argc, char** args)
   for (;;)
   {
     std::string doc = stats->randomSentence(rand() % 45 + 5);
-    std::string hi = vars->parse(doc);
+    std::string hi = doc;
     hi.resize(140);
     
     std::string replyMsg;
@@ -0,0 +1,845 @@
+😄
+😃
+😀
+😊
+☺️
+😉
+😍
+😘
+😚
+😗
+😙
+😜
+😝
+😛
+😳
+😁
+😔
+😌
+😒
+😞
+😣
+😢
+😂
+😭
+😪
+😥
+😰
+😅
+😓
+😩
+😫
+😨
+😱
+😠
+😡
+😤
+😖
+😆
+😋
+😷
+😎
+😴
+😵
+😲
+😟
+😦
+😧
+😈
+👿
+😮
+😬
+😐
+😕
+😯
+😶
+😇
+😏
+😑
+👲
+👳
+👮
+👷
+💂
+👶
+👦
+👧
+👨
+👩
+👴
+👵
+👱
+👼
+👸
+😺
+😸
+😻
+😽
+😼
+🙀
+😿
+😹
+😾
+👹
+👺
+🙈
+🙉
+🙊
+💀
+👽
+💩
+🔥
+✨
+🌟
+💫
+💥
+💢
+💦
+💧
+💤
+💨
+👂
+👀
+👃
+👅
+👄
+👍
+👎
+👌
+👊
+✊
+✌️
+👋
+✋
+👐
+👆
+👇
+👉
+👈
+🙌
+🙏
+☝️
+👏
+💪
+🚶
+🏃
+💃
+👫
+👪
+👬
+👭
+💏
+💑
+👯
+🙆
+🙅
+💁
+🙋
+💆
+💇
+💅
+👰
+🙎
+🙍
+🙇
+🎩
+👑
+👒
+👟
+👞
+👡
+👠
+👢
+👕
+👔
+👚
+👗
+🎽
+👖
+👘
+👙
+💼
+👜
+👝
+👛
+👓
+🎀
+🌂
+💄
+💛
+💙
+💜
+💚
+❤️
+💔
+💗
+💓
+💕
+💖
+💞
+💘
+💌
+💋
+💍
+💎
+👤
+👥
+💬
+👣
+💭
+🐶
+🐺
+🐱
+🐭
+🐹
+🐰
+🐸
+🐯
+🐨
+🐻
+🐷
+🐽
+🐮
+🐗
+🐵
+🐒
+🐴
+🐑
+🐘
+🐼
+🐧
+🐦
+🐤
+🐥
+🐣
+🐔
+🐍
+🐢
+🐛
+🐝
+🐜
+🐞
+🐌
+🐙
+🐚
+🐠
+🐟
+🐬
+🐳
+🐋
+🐄
+🐏
+🐀
+🐃
+🐅
+🐇
+🐉
+🐎
+🐐
+🐓
+🐕
+🐖
+🐁
+🐂
+🐲
+🐡
+🐊
+🐫
+🐪
+🐆
+🐈
+🐩
+🐾
+💐
+🌸
+🌷
+🍀
+🌹
+🌻
+🌺
+🍁
+🍃
+🍂
+🌿
+🌾
+🍄
+🌵
+🌴
+🌲
+🌳
+🌰
+🌱
+🌼
+🌐
+🌞
+🌝
+🌚
+🌑
+🌒
+🌓
+🌔
+🌕
+🌖
+🌗
+🌘
+🌜
+🌛
+🌙
+🌍
+🌎
+🌏
+🌋
+🌌
+🌠
+⭐
+☀️
+⛅
+☁️
+⚡
+☔
+❄️
+⛄
+🌀
+🌁
+🌈
+🌊
+🎍
+💝
+🎎
+🎒
+🎓
+🎏
+🎆
+🎇
+🎐
+🎑
+🎃
+👻
+🎅
+🎄
+🎁
+🎋
+🎉
+🎊
+🎈
+🎌
+🔮
+🎥
+📷
+📹
+📼
+💿
+📀
+💽
+💾
+💻
+📱
+☎️
+📞
+📟
+📠
+📡
+📺
+📻
+🔊
+🔉
+🔈
+🔇
+🔔
+🔕
+📢
+📣
+⏳
+⌛
+⏰
+⌚
+🔓
+🔒
+🔏
+🔐
+🔑
+🔎
+💡
+🔦
+🔆
+🔅
+🔌
+🔋
+🔍
+🛁
+🛀
+🚿
+🚽
+🔧
+🔩
+🔨
+🚪
+🚬
+💣
+🔫
+🔪
+💊
+💉
+💰
+💴
+💵
+💷
+💶
+💳
+💸
+📲
+📧
+📥
+📤
+✉️
+📩
+📨
+📯
+📫
+📪
+📬
+📭
+📮
+📦
+📝
+📄
+📃
+📑
+📊
+📈
+📉
+📜
+📋
+📅
+📆
+📇
+📁
+📂
+✂️
+📌
+📎
+✒️
+✏️
+📏
+📐
+📕
+📗
+📘
+📙
+📓
+📔
+📒
+📚
+📖
+🔖
+📛
+🔬
+🔭
+📰
+🎨
+🎬
+🎤
+🎧
+🎼
+🎵
+🎶
+🎹
+🎻
+🎺
+🎷
+🎸
+👾
+🎮
+🃏
+🎴
+🀄
+🎲
+🎯
+🏈
+🏀
+⚽
+⚾️
+🎾
+🎱
+🏉
+🎳
+⛳
+🚵
+🚴
+🏁
+🏇
+🏆
+🎿
+🏂
+🏊
+🏄
+🎣
+☕
+🍵
+🍶
+🍼
+🍺
+🍻
+🍸
+🍹
+🍷
+🍴
+🍕
+🍔
+🍟
+🍗
+🍖
+🍝
+🍛
+🍤
+🍱
+🍣
+🍥
+🍙
+🍘
+🍚
+🍜
+🍲
+🍢
+🍡
+🍳
+🍞
+🍩
+🍮
+🍦
+🍨
+🍧
+🎂
+🍰
+🍪
+🍫
+🍬
+🍭
+🍯
+🍎
+🍏
+🍊
+🍋
+🍒
+🍇
+🍉
+🍓
+🍑
+🍈
+🍌
+🍐
+🍍
+🍠
+🍆
+🍅
+🌽
+🏠
+🏡
+🏫
+🏢
+🏣
+🏥
+🏦
+🏪
+🏩
+🏨
+💒
+⛪
+🏬
+🏤
+🌇
+🌆
+🏯
+🏰
+⛺
+🏭
+🗼
+🗾
+🗻
+🌄
+🌅
+🌃
+🗽
+🌉
+🎠
+🎡
+⛲
+🎢
+🚢
+⛵
+🚤
+🚣
+⚓
+🚀
+✈️
+💺
+🚁
+🚂
+🚊
+🚉
+🚞
+🚆
+🚄
+🚅
+🚈
+🚇
+🚝
+🚋
+🚃
+🚎
+🚌
+🚍
+🚙
+🚘
+🚗
+🚕
+🚖
+🚛
+🚚
+🚨
+🚓
+🚔
+🚒
+🚑
+🚐
+🚲
+🚡
+🚟
+🚠
+🚜
+💈
+🚏
+🎫
+🚦
+🚥
+⚠️
+🚧
+🔰
+⛽
+🏮
+🎰
+♨️
+🗿
+🎪
+🎭
+📍
+🚩
+🇯🇵
+🇰🇷
+🇩🇪
+🇨🇳
+🇺🇸
+🇫🇷
+🇪🇸
+🇮🇹
+🇷🇺
+🇬🇧
+1️⃣
+2️⃣
+3️⃣
+4️⃣
+5️⃣
+6️⃣
+7️⃣
+8️⃣
+9️⃣
+0️⃣
+🔟
+🔢
+#️⃣
+🔣
+⬆️
+⬇️
+⬅️
+➡️
+🔠
+🔡
+🔤
+↗️
+↖️
+↘️
+↙️
+↔️
+↕️
+🔄
+◀️
+▶️
+🔼
+🔽
+↩️
+↪️
+ℹ️
+⏪
+⏩
+⏫
+⏬
+⤵️
+⤴️
+🆗
+🔀
+🔁
+🔂
+🆕
+🆙
+🆒
+🆓
+🆖
+📶
+🎦
+🈁
+🈯
+🈳
+🈵
+🈴
+🈲
+🉐
+🈹
+🈺
+🈶
+🈚
+🚻
+🚹
+🚺
+🚼
+🚾
+🚰
+🚮
+🅿️
+♿
+🚭
+🈷️
+🈸
+🈂️
+Ⓜ️
+🛂
+🛄
+🛅
+🛃
+🉑
+㊙️
+㊗️
+🆑
+🆘
+🆔
+🚫
+🔞
+📵
+🚯
+🚱
+🚳
+🚷
+🚸
+⛔
+✳️
+❇️
+❎
+✅
+✴️
+💟
+🆚
+📳
+📴
+🅰️
+🅱️
+🆎
+🅾️
+💠
+➿
+♻️
+♈
+♉
+♊
+♋
+♌
+♍
+♎
+♏
+♐
+♑
+♒
+♓
+⛎
+🔯
+🏧
+💹
+💲
+💱
+©️
+®️
+™️
+❌
+‼️
+⁉️
+❗
+❓
+❕
+❔
+⭕
+🔝
+🔚
+🔙
+🔛
+🔜
+🔃
+🕛
+🕧
+🕐
+🕜
+🕑
+🕝
+🕒
+🕞
+🕓
+🕟
+🕔
+🕠
+🕕
+🕖
+🕗
+🕘
+🕙
+🕚
+🕡
+🕢
+🕣
+🕤
+🕥
+🕦
+✖️
+➕
+➖
+➗
+♠️
+♥️
+♣️
+♦️
+💮
+💯
+✔️
+☑️
+🔘
+🔗
+➰
+〰️
+〽️
+🔱
+◼️
+◻️
+◾
+◽
+▪️
+▫️
+🔺
+🔲
+🔳
+⚫
+⚪
+🔴
+🔵
+🔻
+⬜
+⬛
+🔶
+🔷
+🔸
+🔹
\ No newline at end of file
@@ -0,0 +1,6 @@
+:)
+:p
+o_o
+:d
+;)
+:o
\ No newline at end of file
@@ -1,46 +1,32 @@
 #include "freevars.h"
 #include <fstream>
-#include <cstdlib>
+#include "kgramstats.h"
 
-freevars::freevars()
+freevar::freevar(word& w, std::string file) : w(w)
 {
-    vars = new std::map<std::string, std::vector<std::string>* >();
+  std::ifstream infile(file);
+  if (infile)
+  {
+    std::string line;
+    while (getline(infile, line))
+    {
+      instances.insert(line);
+      w.forms.add(line);
+    }
+  }
 }
 
-void freevars::addVar(std::string name, std::string filename)
+bool freevar::check(std::string f) const
 {
-    std::vector<std::string>* eltlist = new std::vector<std::string>();
-    
-    std::ifstream infile(filename.c_str());
-    if (infile)
-    {
-        std::string line;
-        
-        while (getline(infile, line))
-        {
-            eltlist->push_back(line);
-        }
-    } else {
-        eltlist->push_back("");
-    }
-    
-    (*vars)[name] = eltlist;
+  return (instances.count(f) == 1);
 }
 
-std::string freevars::parse(std::string in)
+void freevar::add(std::string f)
 {
-    std::string res(in);
-    
-    for (std::map<std::string, std::vector<std::string>* >::iterator it = vars->begin(); it != vars->end(); it++)
-    {
-        std::string tofind = "$" + it->first + "$";
-        size_t fpos;
-        while ((fpos = res.find(tofind)) != std::string::npos)
-        {
-            int r = rand() % it->second->size();
-            res.replace(fpos, tofind.length(), (*it->second)[r], 0, std::string::npos);
-        }
-    }
-    
-    return res;
-}
\ No newline at end of file
+  instances.insert(f);
+}
+
+word& freevar::getWord()
+{
+  return w;
+}
@@ -1,19 +1,22 @@
-#include <map>
 #include <string>
-#include <vector>
+#include <set>
 
 #ifndef FREEVARS_H
 #define FREEVARS_H
 
-class freevars
+class word;
+
+class freevar
 {
-public:
-    freevars();
-    void addVar(std::string name, std::string filename);
-    std::string parse(std::string in);
+  public:
+    freevar(word& w, std::string file);
+    bool check(std::string f) const;
+    void add(std::string f);
+    word& getWord();
     
-private:
-    std::map<std::string, std::vector<std::string>* >* vars;
+  private:
+    word& w;
+    std::set<std::string> instances;
 };
 
 #endif
\ No newline at end of file
@@ -7,7 +7,6 @@
 #include <cstdlib>
 #include <fstream>
 #include <iostream>
-#include "freevars.h"
 
 int main(int argc, char** args)
 {
@@ -44,16 +43,11 @@ int main(int argc, char** args)
   std::cout << "Preprocessing corpus..." << std::endl;
   kgramstats* stats = new kgramstats(corpus, 4);
     
-  std::cout << "Preprocessing freevars..." << std::endl;
-  freevars* vars = new freevars();
-  vars->addVar("name", "names.txt");
-  vars->addVar("noun", "nouns.txt");
-    
   std::cout << "Generating..." << std::endl;
   for (;;)
   {
     std::string doc = stats->randomSentence(rand() % 35 + 15);
-    std::string hi = vars->parse(doc);
+    std::string hi = doc;
     hi.resize(140);
 
     std::cout << hi << std::endl;
@@ -1,5 +1,6 @@
 #include "histogram.h"
 #include <cstdlib>
+#include <iostream>
 
 template <class T>
 void histogram<T>::add(const T& inst)
@@ -31,4 +32,13 @@ const T& histogram<T>::next() const
   return distribution.upper_bound(r)->second;
 }
 
+template <class T>
+void histogram<T>::print() const
+{
+  for (auto& freqpair : freqtable)
+  {
+    std::cout << freqpair.first << ": " << freqpair.second << std::endl;
+  }
+}
+
 template class histogram <std::string>;
@@ -10,6 +10,7 @@ class histogram {
     void add(const T& inst);
     void compile();
     const T& next() const;
+    void print() const;
     
   private:
     std::map<T, int> freqtable;
@@ -39,6 +39,9 @@
 #include <algorithm>
 #include <set>
 #include <stack>
+#include "freevars.h"
+#include <fstream>
+#include "prefix_search.h"
 
 query wildcardQuery {querytype::sentence};
 word blank_word {""};
@@ -53,14 +56,55 @@ kgramstats::kgramstats(std::string corpus, int maxK)
   size_t start = 0;
   int end = 0;
   std::set<std::string> thashtags;
+  freevar fv_emoticons {emoticons, "emoticons.txt"};
+  
+  std::cout << "Reading emojis..." << std::endl;
+  prefix_search emojis;
+  std::ifstream emoji_file("emojis.txt");
+  if (emoji_file)
+  {
+    while (!emoji_file.eof())
+    {
+      std::string rawmojis;
+      getline(emoji_file, rawmojis);
+      emojis.add(rawmojis);
+    }
+    
+    emoji_file.close();
+  }
 
+  std::cout << "Tokenizing corpus..." << std::endl;
   while (end != std::string::npos)
   {
     end = corpus.find(" ", start);
 
-    std::string t = corpus.substr(start, (end == std::string::npos) ? std::string::npos : end - start);
-    if (t.compare("") && t.compare("."))
+    bool emoji = false;
+    std::string te = corpus.substr(start, (end == std::string::npos) ? std::string::npos : end - start);
+    std::string t = "";
+    
+    if (te.compare("") && te.compare("."))
     {
+      // Extract strings of emojis into their own tokens even if they're not space delimited
+      int m = emojis.match(te);
+      emoji = m > 0;
+      if (m == 0) m = 1;
+      t = te.substr(0,m);
+      te = te.substr(m);
+      
+      while (!te.empty())
+      {
+        m = emojis.match(te);
+        if (emoji == (m > 0))
+        {
+          if (m == 0) m = 1;
+          t += te.substr(0,m);
+          te = te.substr(m);
+        } else {
+          end = start + t.length() - 1;
+          break;
+        }
+      }
+      
       std::string tc(t), canonical;
       std::transform(tc.begin(), tc.end(), tc.begin(), ::tolower);
       std::remove_copy_if(tc.begin(), tc.end(), std::back_inserter(canonical), [=] (char c) {
@@ -72,11 +116,29 @@ kgramstats::kgramstats(std::string corpus, int maxK)
         if (canonical[0] == '#')
         {
           thashtags.insert(canonical);
-          canonical = "#hashtag";
           
           return hashtags;
         }
         
+        // Emoticon freevar
+        if (emoji)
+        {
+          emoticons.forms.add(canonical);
+          
+          return emoticons;
+        }
+        
+        std::string emoticon_canon;
+        std::remove_copy_if(tc.begin(), tc.end(), std::back_inserter(emoticon_canon), [=] (char c) {
+          return !((c != '.') && (c != '?') && (c != '!') && (c != ',') && (c != '"') && (c != '\n') && (c != '[') && (c != ']') && (c != '*'));
+        });
+        if (fv_emoticons.check(emoticon_canon))
+        {
+          emoticons.forms.add(emoticon_canon);
+          
+          return emoticons;
+        }
+        
         // Basically any other word
         if (words.count(canonical) == 0)
         {
@@ -171,6 +233,7 @@ kgramstats::kgramstats(std::string corpus, int maxK)
   }
   
   // Time to condense the distribution stuff for the words
+  std::cout << "Compiling token histograms..." << std::endl;
   for (auto& it : words)
   {
     it.second.forms.compile();
@@ -185,8 +248,13 @@ kgramstats::kgramstats(std::string corpus, int maxK)
   
   hashtags.forms.compile();
   hashtags.terms.compile();
+  
+  // Compile other freevars
+  emoticons.forms.compile();
+  emoticons.terms.compile();
 
   // kgram distribution
+  std::cout << "Creating markov chain..." << std::endl;
   std::map<kgram, std::map<token, token_data> > tstats;
   for (int k=1; k<maxK; k++)
   {
@@ -246,6 +314,7 @@ kgramstats::kgramstats(std::string corpus, int maxK)
   }
         
   // Condense the kgram distribution
+  std::cout << "Compiling kgram distributions..." << std::endl;
   for (auto& it : tstats)
   {
     kgram klist = it.first;
@@ -454,6 +523,36 @@ std::string kgramstats::randomSentence(int n)
     
     open_delimiters.pop();
   }
+  
+  // Replace old-style freevars while I can't be bothered to remake the corpus yet
+  std::vector<std::string> fv_names;
+  std::ifstream namefile("names.txt");
+  while (!namefile.eof())
+  {
+    std::string l;
+    getline(namefile, l);
+    fv_names.push_back(l);
+  }
+  
+  int cpos;
+  while ((cpos = result.find("$name$")) != std::string::npos)
+  {
+    result.replace(cpos, 6, fv_names[rand() % fv_names.size()]);
+  }
+  
+  std::vector<std::string> fv_nouns;
+  std::ifstream nounfile("nouns.txt");
+  while (!nounfile.eof())
+  {
+    std::string l;
+    getline(nounfile, l);
+    fv_nouns.push_back(l);
+  }
+  
+  while ((cpos = result.find("$noun$")) != std::string::npos)
+  {
+    result.replace(cpos, 6, fv_nouns[rand() % fv_nouns.size()]);
+  }
         
   return result;
 }
@@ -112,8 +112,11 @@ private:
   
         int maxK;
         std::map<kgram, std::map<int, token_data> > stats;
-  word hashtags {"#hashtag"};
+  
+  // Words
   std::map<std::string, word> words;
+  word hashtags {"#hashtag"};
+  word emoticons {"👌"};
 };
 
 void printKgram(kgram k);
@@ -0,0 +1,35 @@
+#include "prefix_search.h"
+
+void prefix_search::add(std::string prefix)
+{
+  node* cur = &top;
+  for (int c : prefix)
+  {
+    cur = &cur->children[c];
+  }
+  
+  cur->match = true;
+}
+
+int prefix_search::match(std::string in) const
+{
+  int ret = 0;
+  const node* cur = &top;
+  for (int c : in)
+  {
+    if (cur->children.count(c) == 0)
+    {
+      return 0;
+    }
+    
+    cur = &cur->children.at(c);
+    ret++;
+    
+    if (cur->match)
+    {
+      return ret;
+    }
+  }
+  
+  return 0;
+}
@@ -0,0 +1,23 @@
+#ifndef PREFIX_SEARCH_H_5CFCF783
+#define PREFIX_SEARCH_H_5CFCF783
+
+#include <map>
+#include <string>
+
+class prefix_search {
+  public:
+    void add(std::string prefix);
+    int match(std::string in) const;
+    
+  private:
+    struct node {
+      std::map<int, struct node> children;
+      bool match;
+      
+      node() : match(false) {}
+    };
+    
+    node top;
+};
+
+#endif /* end of include guard: PREFIX_SEARCH_H_5CFCF783 */