Change output format to zstd compressed Godot serialized variant

1 files changed, 113 insertions, 46 deletions

@@ -3,6 +3,7 @@
 #include <fmt/core.h>
 #include <hkutil/progress.h>
 #include <hkutil/string.h>
+#include <zstd.h>
 
 #include <algorithm>
 #include <filesystem>
@@ -10,13 +11,17 @@
 #include <list>
 #include <regex>
 #include <set>
+#include <sstream>
 #include <stdexcept>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
+#include "godot_variant.h"
+
 constexpr int MIN_FREQUENCY = 2000000;
+constexpr int kCompressionLevel = 3;
 
 namespace {
 
@@ -877,7 +882,7 @@ void generator::run() {
   }
 
   // Orange addition
-  std::vector<std::string> orange_addition;
+  std::vector<GodotVariant> orange_addition;
   {
     hatkirby::progress ppgs("Generating orange addition puzzles...",
                             wanderlust_.size());
@@ -894,8 +899,12 @@ void generator::run() {
           continue;
         }
         if (wanderlust_.count(cipher - cipher2)) {
-          orange_addition.push_back(fmt::format(
-              "[{},{},{}]", form_id2, wanderlust_[cipher - cipher2], form_id));
+          std::vector<GodotVariant> addition_entry;
+          addition_entry.emplace_back(static_cast<int32_t>(form_id2));
+          addition_entry.emplace_back(
+              static_cast<int32_t>(wanderlust_[cipher - cipher2]));
+          addition_entry.emplace_back(static_cast<int32_t>(form_id));
+          orange_addition.emplace_back(addition_entry);
         }
       }
     }
@@ -995,83 +1004,141 @@ void generator::run() {
   std::cout << "Yellow top yellow middle combos: "
             << combos_[kYellowTop][kYellowMiddle].size() << std::endl;
 
-  std::vector<std::string> form_entry;
+  // 0: forms
+  // 1: paintings
+  // 2: wanderlust
+  // 3: addition
+  // 4: walls
+  // 5: combos
+  std::vector<GodotVariant> full_variant;
+
+  std::vector<GodotVariant> form_entry;
   form_entry.reserve(forms_.size());
   for (const Form& form : forms_) {
     if (form.puzzles.empty()) {
-      form_entry.push_back(fmt::format("\"{}\"", form.text));
+      form_entry.emplace_back(form.text);
     } else {
-      std::vector<std::string> entry_per_type;
+      std::map<GodotVariant, GodotVariant> entry_per_type;
       for (const auto& [puzzle_type, puzzles] : form.puzzles) {
-        std::vector<std::string> entry_per_puzzle;
+        std::vector<GodotVariant> entry_per_puzzle;
         for (size_t puzzle : puzzles) {
-          entry_per_puzzle.push_back(std::to_string(puzzle));
+          entry_per_puzzle.emplace_back(puzzle);
         }
-        entry_per_type.push_back(
-            fmt::format("{}:[{}]", static_cast<int>(puzzle_type),
-                        hatkirby::implode(entry_per_puzzle, ",")));
+        entry_per_type.emplace(
+            std::piecewise_construct,
+            std::forward_as_tuple(static_cast<int32_t>(puzzle_type)),
+            std::forward_as_tuple(entry_per_puzzle));
+      }
+      std::vector<GodotVariant> pair_entry;
+      pair_entry.emplace_back(form.text);
+      if (!entry_per_type.empty()) {
+        pair_entry.emplace_back(entry_per_type);
       }
-      form_entry.push_back(fmt::format("[\"{}\",{{{}}}]", form.text,
-                                       hatkirby::implode(entry_per_type, ",")));
+      form_entry.emplace_back(pair_entry);
     }
   }
+  full_variant.emplace_back(form_entry);
 
-  std::ofstream output_file(outputPath_);
-  output_file << "extends Node\n\nvar forms = ["
-              << hatkirby::implode(form_entry, ",") << "]" << std::endl;
-
-  std::vector<std::string> painting_entries;
+  std::vector<GodotVariant> painting_entries;
   {
     std::list<std::string> paintings(readFile(datadirPath_ / "paintings.txt"));
     for (const std::string& line : paintings) {
       auto parts = hatkirby::split<std::vector<std::string>>(line, ",");
-      painting_entries.push_back(
-          fmt::format("[\"{}\",\"{}\"]", parts[0], parts[1]));
+      std::vector<GodotVariant> painting_entry;
+      painting_entry.emplace_back(parts[0]);
+      painting_entry.emplace_back(parts[1]);
+      painting_entries.emplace_back(painting_entry);
     }
   }
-  output_file << "var paintings = [" << hatkirby::implode(painting_entries, ",")
-              << "]" << std::endl;
+  full_variant.emplace_back(painting_entries);
 
-  std::vector<std::string> cipher_lines;
+  std::vector<GodotVariant> wanderlust_entries;
   for (const auto& [cipher, form_id] : wanderlust_) {
-    cipher_lines.push_back(std::to_string(form_id));
+    wanderlust_entries.emplace_back(static_cast<int32_t>(form_id));
   }
-  output_file << "var wanderlust = [" << hatkirby::implode(cipher_lines, ",")
-              << "]" << std::endl;
-  output_file << "var addition = [" << hatkirby::implode(orange_addition, ",")
-              << "]" << std::endl;
+  full_variant.emplace_back(wanderlust_entries);
+
+  full_variant.emplace_back(orange_addition);
 
-  std::vector<std::string> walls_entries;
+  std::vector<GodotVariant> walls_entries;
   {
     std::list<std::string> walls(readFile(datadirPath_ / "walls.txt"));
     for (const std::string& line : walls) {
       auto parts = hatkirby::split<std::vector<std::string>>(line, ",");
-      walls_entries.push_back(
-          fmt::format("[\"{}\",\"{}\"]", parts[0], parts[1]));
+      std::vector<GodotVariant> walls_entry;
+      walls_entry.emplace_back(parts[0]);
+      walls_entry.emplace_back(parts[1]);
+      walls_entries.emplace_back(walls_entry);
     }
   }
-  output_file << "var walls_puzzles = ["
-              << hatkirby::implode(walls_entries, ",") << "]" << std::endl;
+  full_variant.emplace_back(walls_entries);
 
-  std::vector<std::string> combo_entries;
+  std::map<GodotVariant, GodotVariant> combo_entries;
   for (const auto& [left_type, left_join] : combos_) {
-    std::vector<std::string> left_entries;
+    std::map<GodotVariant, GodotVariant> left_entries;
     for (const auto& [right_type, choices] : left_join) {
-      std::vector<std::string> choice_entries;
+      std::vector<GodotVariant> choice_entries;
       for (const auto& [hint1, hint2, answer] : choices) {
-        choice_entries.push_back(
-            fmt::format("[{},{},{}]", hint1, hint2, answer));
+        std::vector<GodotVariant> choice_entry;
+        choice_entry.emplace_back(hint1);
+        choice_entry.emplace_back(hint2);
+        choice_entry.emplace_back(answer);
+        choice_entries.emplace_back(choice_entry);
       }
-      left_entries.push_back(
-          fmt::format("{}:[{}]", static_cast<int>(right_type),
-                      hatkirby::implode(choice_entries, ",")));
+      left_entries.emplace(
+          std::piecewise_construct,
+          std::forward_as_tuple(static_cast<int32_t>(right_type)),
+          std::forward_as_tuple(choice_entries));
     }
-    combo_entries.push_back(fmt::format("{}:{{{}}}",
-                                        static_cast<int>(left_type),
-                                        hatkirby::implode(left_entries, ",")));
+    combo_entries.emplace(
+        std::piecewise_construct,
+        std::forward_as_tuple(static_cast<int32_t>(left_type)),
+        std::forward_as_tuple(left_entries));
+  }
+  full_variant.emplace_back(combo_entries);
+
+  std::ostringstream serialized_variant_buffer;
+  GodotVariant(full_variant).Serialize(serialized_variant_buffer);
+
+  std::string serialized_variant = serialized_variant_buffer.str();
+  std::ostringstream variant_file_buffer;
+  int32_t variant_len = serialized_variant.size();
+  variant_file_buffer.write(reinterpret_cast<char*>(&variant_len), 4);
+  variant_file_buffer.write(serialized_variant.data(),
+                            serialized_variant.size());
+
+  std::string variant_file = variant_file_buffer.str();
+
+  size_t max_output_size = ZSTD_compressBound(variant_file.size());
+  std::string compressed_variant(max_output_size, 0);
+  size_t compressed_size = ZSTD_compress(
+      compressed_variant.data(), max_output_size, variant_file.data(),
+      variant_file.size(), kCompressionLevel);
+
+  // Write Godot's weird gzip container
+  {
+    std::ofstream output_file(outputPath_, std::ios::binary);
+
+    int32_t bytes = 0x46504347;
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    bytes = 0x00000002;  // ZSTD
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    bytes = variant_file.size() + 1;
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    bytes = variant_file.size();
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    bytes = compressed_size;
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    output_file.write(compressed_variant.data(), compressed_size);
+
+    bytes = 0x46504347;
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
   }
-  output_file << "var combos = {" << hatkirby::implode(combo_entries, ",")
-              << "}" << std::endl;
 }
 
 size_t generator::LookupOrCreatePronunciation(const std::string& phonemes) {