Newlines, colons, and semicolons are now valid terminators

5 files changed, 72 insertions, 27 deletions

@@ -37,7 +37,7 @@ int main(int argc, char** args)
       line.pop_back();
     }
     
-    corpus += line + "\n ";
+    corpus += line + "\n";
   }
   
   // Replace old-style freevars while I can't be bothered to remake the corpus yet
@@ -42,7 +42,7 @@ int main(int argc, char** args)
       line.pop_back();
     }
     
-    corpus += line + "\n ";
+    corpus += line + "\n";
   }
   
   // Replace old-style freevars while I can't be bothered to remake the corpus yet
@@ -1,4 +1,5 @@
 #include "histogram.h"
+#include "kgramstats.h"
 #include <cstdlib>
 #include <iostream>
 
@@ -42,3 +43,4 @@ void histogram<T>::print() const
 }
 
 template class histogram <std::string>;
+template class histogram <rawr::terminator>;
@@ -133,14 +133,19 @@ void rawr::compile(int maxK)
         std::cout << per << "%" << std::flush;
       }
     
-      end = _corpora[i].find(" ", start);
+      end = _corpora[i].find_first_of(" \n", start);
 
       bool emoji = false;
-      std::string te = _corpora[i].substr(start, (end == std::string::npos) ? std::string::npos : end - start);
+      std::string te = _corpora[i].substr(start, (end == std::string::npos) ? std::string::npos : end - start + 1);
       std::string t = "";
     
-      if (te.compare("") && te.compare("."))
+      if (te.compare("") && te.compare(".") && te.compare(" "))
       {
+        if (te.back() == ' ')
+        {
+          te.pop_back();
+        }
+        
         // Extract strings of emojis into their own tokens even if they're not space delimited
         int m = emojis.match(te);
         emoji = m > 0;
@@ -166,7 +171,7 @@ void rawr::compile(int maxK)
         std::transform(tc.begin(), tc.end(), tc.begin(), ::tolower);
 
         int pst = tc.find_first_not_of("\"([*");
-        int dst = tc.find_last_not_of("\")]*.,?!\n");
+        int dst = tc.find_last_not_of("\")]*.,?!\n;:");
         std::string canonical("");
         if ((pst != std::string::npos) && (dst != std::string::npos))
         {
@@ -270,28 +275,28 @@ void rawr::compile(int maxK)
           }
         }
       
-        int backtrack = t.find_last_not_of(".,?!])\"*\n") + 1;
+        int backtrack = t.find_last_not_of(".,?!])\"*\n;:") + 1;
         if (backtrack != t.length())
         {
           std::string ending = t.substr(backtrack);
           std::string suffix;
+          bool newline = false;
+          bool terminating = false;
         
           for (char c : ending)
           {
-            if ((c == '.') || (c == ',') || (c == '?') || (c == '!'))
+            if ((c == '.') || (c == ',') || (c == '?') || (c == '!') || (c == ';') || (c == ':'))
             {
               suffix += c;
+              terminating = true;
             
               continue;
             } else if (c == '\n')
             {
-              // At least the end is coming
-              if (suffix.empty())
-              {
-                suffix = ".";
-              }
-            
-              break;
+              newline = true;
+              terminating = true;
+              
+              continue;
             }
           
             parentype pt = ([&] {
@@ -313,13 +318,16 @@ void rawr::compile(int maxK)
             }
           }
         
-          if (suffix == ",")
+          if (terminating)
           {
-            tk.suffix = suffixtype::comma;
-          } else if (!suffix.empty()) {
-            tk.suffix = suffixtype::terminating;
-          
-            w.terms.add(suffix);
+            if ((suffix == ",") && (!newline))
+            {
+              tk.suffix = suffixtype::comma;
+            } else {
+              tk.suffix = suffixtype::terminating;
+              
+              w.terms.add({suffix, newline});
+            }
           }
         }
       
@@ -502,6 +510,18 @@ std::ostream& operator<<(std::ostream& os, rawr::token t)
   }
 }
 
+std::ostream& operator<<(std::ostream& os, rawr::terminator t)
+{
+  os << t.form;
+  
+  if (t.newline)
+  {
+    os << "↵";
+  }
+  
+  return os;
+}
+
 void rawr::setTransformCallback(transform_callback _arg)
 {
   _transform = _arg;
@@ -649,10 +669,20 @@ std::string rawr::randomSentence(int maxL)
     // Terminators
     if (next.tok.suffix == suffixtype::terminating)
     {
-      nextToken.append(next.tok.w.terms.next());
+      auto term = next.tok.w.terms.next();
+      nextToken.append(term.form);
+      
+      if (term.newline)
+      {
+        nextToken.append("\n");
+      } else {
+        nextToken.append(" ");
+      }
     } else if (next.tok.suffix == suffixtype::comma)
     {
-      nextToken.append(",");
+      nextToken.append(", ");
+    } else {
+      nextToken.append(" ");
     }
     
     // If this pick was guaranteed, increase cut chance
@@ -665,9 +695,8 @@ std::string rawr::randomSentence(int maxL)
     std::cout << cur << "-> \"" << nextToken << "\" (" << next.all << "/" << max << ")" << std::endl;
 
     cur.push_back(next.tok);
-                
-    result.append(nextToken + " ");
-    
+    result.append(nextToken);
+        
     if ((next.tok.suffix == suffixtype::terminating) && ((result.length() > maxL) || (rand() % 4 == 0)))
     {
       break;
@@ -19,10 +19,22 @@ class rawr {
         std::string randomSentence(int maxL);
         
   private:
+    struct terminator {
+      std::string form;
+      bool newline = false;
+      
+      terminator(std::string form, bool newline) : form(form), newline(newline) {}
+      
+      bool operator<(const terminator& other) const
+      {
+        return std::tie(form, newline) < std::tie(other.form, other.newline);
+      }
+    };
+    
     struct word {
       std::string canon;
       histogram<std::string> forms;
-      histogram<std::string> terms;
+      histogram<terminator> terms;
   
       word(std::string canon) : canon(canon) {}
   
@@ -68,6 +80,7 @@ class rawr {
       std::map<delimiter, int> delimiters;
       suffixtype suffix;
       std::string raw;
+      bool newline = false;
     
       token(const word& w) : w(w), suffix(suffixtype::none) {}
   
@@ -119,6 +132,7 @@ class rawr {
     friend std::ostream& operator<<(std::ostream& os, kgram k);
     friend std::ostream& operator<<(std::ostream& os, query q);
     friend std::ostream& operator<<(std::ostream& os, token t);
+    friend std::ostream& operator<<(std::ostream& os, terminator t);
   
         int _maxK;
     bool _compiled = false;