Change output format to zstd compressed Godot serialized variant

4 files changed, 217 insertions, 49 deletions

@@ -1,11 +1,15 @@
 cmake_minimum_required (VERSION 3.1)
 project (generator)
 
+find_package(PkgConfig)
+pkg_check_modules(libzstd libzstd REQUIRED)
+
 add_subdirectory(vendor/fmt)
 
-include_directories(vendor/hkutil vendor/fmt/include)
+include_directories(vendor/hkutil vendor/fmt/include ${libzstd_INCLUDE_DIRS})
+link_directories(${libzstd_LIBRARY_DIRS})
 
-add_executable(generator generator.cpp main.cpp)
+add_executable(generator generator.cpp main.cpp godot_variant.cpp)
 set_property(TARGET generator PROPERTY CXX_STANDARD 17)
 set_property(TARGET generator PROPERTY CXX_STANDARD_REQUIRED ON)
-target_link_libraries(generator fmt)
+target_link_libraries(generator fmt ${libzstd_LIBRARIES})
@@ -3,6 +3,7 @@
 #include <fmt/core.h>
 #include <hkutil/progress.h>
 #include <hkutil/string.h>
+#include <zstd.h>
 
 #include <algorithm>
 #include <filesystem>
@@ -10,13 +11,17 @@
 #include <list>
 #include <regex>
 #include <set>
+#include <sstream>
 #include <stdexcept>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
+#include "godot_variant.h"
+
 constexpr int MIN_FREQUENCY = 2000000;
+constexpr int kCompressionLevel = 3;
 
 namespace {
 
@@ -877,7 +882,7 @@ void generator::run() {
   }
 
   // Orange addition
-  std::vector<std::string> orange_addition;
+  std::vector<GodotVariant> orange_addition;
   {
     hatkirby::progress ppgs("Generating orange addition puzzles...",
                             wanderlust_.size());
@@ -894,8 +899,12 @@ void generator::run() {
           continue;
         }
         if (wanderlust_.count(cipher - cipher2)) {
-          orange_addition.push_back(fmt::format(
-              "[{},{},{}]", form_id2, wanderlust_[cipher - cipher2], form_id));
+          std::vector<GodotVariant> addition_entry;
+          addition_entry.emplace_back(static_cast<int32_t>(form_id2));
+          addition_entry.emplace_back(
+              static_cast<int32_t>(wanderlust_[cipher - cipher2]));
+          addition_entry.emplace_back(static_cast<int32_t>(form_id));
+          orange_addition.emplace_back(addition_entry);
         }
       }
     }
@@ -995,83 +1004,141 @@ void generator::run() {
   std::cout << "Yellow top yellow middle combos: "
             << combos_[kYellowTop][kYellowMiddle].size() << std::endl;
 
-  std::vector<std::string> form_entry;
+  // 0: forms
+  // 1: paintings
+  // 2: wanderlust
+  // 3: addition
+  // 4: walls
+  // 5: combos
+  std::vector<GodotVariant> full_variant;
+
+  std::vector<GodotVariant> form_entry;
   form_entry.reserve(forms_.size());
   for (const Form& form : forms_) {
     if (form.puzzles.empty()) {
-      form_entry.push_back(fmt::format("\"{}\"", form.text));
+      form_entry.emplace_back(form.text);
     } else {
-      std::vector<std::string> entry_per_type;
+      std::map<GodotVariant, GodotVariant> entry_per_type;
       for (const auto& [puzzle_type, puzzles] : form.puzzles) {
-        std::vector<std::string> entry_per_puzzle;
+        std::vector<GodotVariant> entry_per_puzzle;
         for (size_t puzzle : puzzles) {
-          entry_per_puzzle.push_back(std::to_string(puzzle));
+          entry_per_puzzle.emplace_back(puzzle);
         }
-        entry_per_type.push_back(
-            fmt::format("{}:[{}]", static_cast<int>(puzzle_type),
-                        hatkirby::implode(entry_per_puzzle, ",")));
+        entry_per_type.emplace(
+            std::piecewise_construct,
+            std::forward_as_tuple(static_cast<int32_t>(puzzle_type)),
+            std::forward_as_tuple(entry_per_puzzle));
+      }
+      std::vector<GodotVariant> pair_entry;
+      pair_entry.emplace_back(form.text);
+      if (!entry_per_type.empty()) {
+        pair_entry.emplace_back(entry_per_type);
       }
-      form_entry.push_back(fmt::format("[\"{}\",{{{}}}]", form.text,
-                                       hatkirby::implode(entry_per_type, ",")));
+      form_entry.emplace_back(pair_entry);
     }
   }
+  full_variant.emplace_back(form_entry);
 
-  std::ofstream output_file(outputPath_);
-  output_file << "extends Node\n\nvar forms = ["
-              << hatkirby::implode(form_entry, ",") << "]" << std::endl;
-
-  std::vector<std::string> painting_entries;
+  std::vector<GodotVariant> painting_entries;
   {
     std::list<std::string> paintings(readFile(datadirPath_ / "paintings.txt"));
     for (const std::string& line : paintings) {
       auto parts = hatkirby::split<std::vector<std::string>>(line, ",");
-      painting_entries.push_back(
-          fmt::format("[\"{}\",\"{}\"]", parts[0], parts[1]));
+      std::vector<GodotVariant> painting_entry;
+      painting_entry.emplace_back(parts[0]);
+      painting_entry.emplace_back(parts[1]);
+      painting_entries.emplace_back(painting_entry);
     }
   }
-  output_file << "var paintings = [" << hatkirby::implode(painting_entries, ",")
-              << "]" << std::endl;
+  full_variant.emplace_back(painting_entries);
 
-  std::vector<std::string> cipher_lines;
+  std::vector<GodotVariant> wanderlust_entries;
   for (const auto& [cipher, form_id] : wanderlust_) {
-    cipher_lines.push_back(std::to_string(form_id));
+    wanderlust_entries.emplace_back(static_cast<int32_t>(form_id));
   }
-  output_file << "var wanderlust = [" << hatkirby::implode(cipher_lines, ",")
-              << "]" << std::endl;
-  output_file << "var addition = [" << hatkirby::implode(orange_addition, ",")
-              << "]" << std::endl;
+  full_variant.emplace_back(wanderlust_entries);
+
+  full_variant.emplace_back(orange_addition);
 
-  std::vector<std::string> walls_entries;
+  std::vector<GodotVariant> walls_entries;
   {
     std::list<std::string> walls(readFile(datadirPath_ / "walls.txt"));
     for (const std::string& line : walls) {
       auto parts = hatkirby::split<std::vector<std::string>>(line, ",");
-      walls_entries.push_back(
-          fmt::format("[\"{}\",\"{}\"]", parts[0], parts[1]));
+      std::vector<GodotVariant> walls_entry;
+      walls_entry.emplace_back(parts[0]);
+      walls_entry.emplace_back(parts[1]);
+      walls_entries.emplace_back(walls_entry);
     }
   }
-  output_file << "var walls_puzzles = ["
-              << hatkirby::implode(walls_entries, ",") << "]" << std::endl;
+  full_variant.emplace_back(walls_entries);
 
-  std::vector<std::string> combo_entries;
+  std::map<GodotVariant, GodotVariant> combo_entries;
   for (const auto& [left_type, left_join] : combos_) {
-    std::vector<std::string> left_entries;
+    std::map<GodotVariant, GodotVariant> left_entries;
     for (const auto& [right_type, choices] : left_join) {
-      std::vector<std::string> choice_entries;
+      std::vector<GodotVariant> choice_entries;
       for (const auto& [hint1, hint2, answer] : choices) {
-        choice_entries.push_back(
-            fmt::format("[{},{},{}]", hint1, hint2, answer));
+        std::vector<GodotVariant> choice_entry;
+        choice_entry.emplace_back(hint1);
+        choice_entry.emplace_back(hint2);
+        choice_entry.emplace_back(answer);
+        choice_entries.emplace_back(choice_entry);
       }
-      left_entries.push_back(
-          fmt::format("{}:[{}]", static_cast<int>(right_type),
-                      hatkirby::implode(choice_entries, ",")));
+      left_entries.emplace(
+          std::piecewise_construct,
+          std::forward_as_tuple(static_cast<int32_t>(right_type)),
+          std::forward_as_tuple(choice_entries));
     }
-    combo_entries.push_back(fmt::format("{}:{{{}}}",
-                                        static_cast<int>(left_type),
-                                        hatkirby::implode(left_entries, ",")));
+    combo_entries.emplace(
+        std::piecewise_construct,
+        std::forward_as_tuple(static_cast<int32_t>(left_type)),
+        std::forward_as_tuple(left_entries));
+  }
+  full_variant.emplace_back(combo_entries);
+
+  std::ostringstream serialized_variant_buffer;
+  GodotVariant(full_variant).Serialize(serialized_variant_buffer);
+
+  std::string serialized_variant = serialized_variant_buffer.str();
+  std::ostringstream variant_file_buffer;
+  int32_t variant_len = serialized_variant.size();
+  variant_file_buffer.write(reinterpret_cast<char*>(&variant_len), 4);
+  variant_file_buffer.write(serialized_variant.data(),
+                            serialized_variant.size());
+
+  std::string variant_file = variant_file_buffer.str();
+
+  size_t max_output_size = ZSTD_compressBound(variant_file.size());
+  std::string compressed_variant(max_output_size, 0);
+  size_t compressed_size = ZSTD_compress(
+      compressed_variant.data(), max_output_size, variant_file.data(),
+      variant_file.size(), kCompressionLevel);
+
+  // Write Godot's weird gzip container
+  {
+    std::ofstream output_file(outputPath_, std::ios::binary);
+
+    int32_t bytes = 0x46504347;
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    bytes = 0x00000002;  // ZSTD
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    bytes = variant_file.size() + 1;
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    bytes = variant_file.size();
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    bytes = compressed_size;
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
+
+    output_file.write(compressed_variant.data(), compressed_size);
+
+    bytes = 0x46504347;
+    output_file.write(reinterpret_cast<char*>(&bytes), 4);
   }
-  output_file << "var combos = {" << hatkirby::implode(combo_entries, ",")
-              << "}" << std::endl;
 }
 
 size_t generator::LookupOrCreatePronunciation(const std::string& phonemes) {
@@ -0,0 +1,69 @@
+#include "godot_variant.h"
+
+#include <stdexcept>
+
+bool GodotVariant::operator<(const GodotVariant& rhs) const {
+  if (value.index() != rhs.value.index()) {
+    return value.index() < rhs.value.index();
+  }
+
+  if (const int32_t* pval = std::get_if<int32_t>(&value)) {
+    return *pval < std::get<int32_t>(rhs.value);
+  } else if (const std::string* pval = std::get_if<std::string>(&value)) {
+    return *pval < std::get<std::string>(rhs.value);
+  } else if (const std::map<GodotVariant, GodotVariant>* pval =
+                 std::get_if<std::map<GodotVariant, GodotVariant>>(&value)) {
+    return *pval < std::get<std::map<GodotVariant, GodotVariant>>(rhs.value);
+  } else if (const std::vector<GodotVariant>* pval =
+                 std::get_if<std::vector<GodotVariant>>(&value)) {
+    return *pval < std::get<std::vector<GodotVariant>>(rhs.value);
+  }
+
+  throw std::invalid_argument("Incomparable variants.");
+}
+
+void GodotVariant::Serialize(std::basic_ostream<char>& output) const {
+  if (const int32_t* pval = std::get_if<int32_t>(&value)) {
+    uint32_t typebits = 2;
+    output.write(reinterpret_cast<char*>(&typebits), 4);
+    output.write(reinterpret_cast<const char*>(pval), 4);
+  } else if (const std::string* pval = std::get_if<std::string>(&value)) {
+    uint32_t typebits = 4;
+    output.write(reinterpret_cast<char*>(&typebits), 4);
+
+    uint32_t lengthbits = pval->length();
+    output.write(reinterpret_cast<char*>(&lengthbits), 4);
+
+    output.write(pval->data(), pval->length());
+
+    if (pval->length() % 4 != 0) {
+      int padding = 4 - (pval->length() % 4);
+      for (int i = 0; i < padding; i++) {
+        output.put(0);
+      }
+    }
+  } else if (const std::map<GodotVariant, GodotVariant>* pval =
+                 std::get_if<std::map<GodotVariant, GodotVariant>>(&value)) {
+    uint32_t typebits = 18;
+    output.write(reinterpret_cast<char*>(&typebits), 4);
+
+    uint32_t lengthbits = pval->size() & 0x7fffffff;
+    output.write(reinterpret_cast<char*>(&lengthbits), 4);
+
+    for (const auto& mapping : *pval) {
+      mapping.first.Serialize(output);
+      mapping.second.Serialize(output);
+    }
+  } else if (const std::vector<GodotVariant>* pval =
+                 std::get_if<std::vector<GodotVariant>>(&value)) {
+    uint32_t typebits = 19;
+    output.write(reinterpret_cast<char*>(&typebits), 4);
+
+    uint32_t lengthbits = pval->size() & 0x7fffffff;
+    output.write(reinterpret_cast<char*>(&lengthbits), 4);
+
+    for (const auto& entry : *pval) {
+      entry.Serialize(output);
+    }
+  }
+}
@@ -0,0 +1,28 @@
+#ifndef GODOT_VARIANT_H_84682779
+#define GODOT_VARIANT_H_84682779
+
+#include <cctype>
+#include <map>
+#include <ostream>
+#include <string>
+#include <variant>
+#include <vector>
+
+struct GodotVariant {
+  using variant_type = std::variant<std::monostate, int32_t, std::string,
+                                    std::map<GodotVariant, GodotVariant>,
+                                    std::vector<GodotVariant> >;
+
+  variant_type value;
+
+  GodotVariant(int32_t v) : value(v) {}
+  GodotVariant(std::string v) : value(std::move(v)) {}
+  GodotVariant(std::map<GodotVariant, GodotVariant> v) : value(std::move(v)) {}
+  GodotVariant(std::vector<GodotVariant> v) : value(std::move(v)) {}
+
+  bool operator<(const GodotVariant& rhs) const;
+
+  void Serialize(std::basic_ostream<char>& output) const;
+};
+
+#endif /* end of include guard: GODOT_VARIANT_H_84682779 */