From d1024b559c44a143eca214fb8732001080e8b037 Mon Sep 17 00:00:00 2001 From: Star Rauchenberger Date: Fri, 6 Dec 2024 17:33:40 -0500 Subject: Change output format to zstd compressed Godot serialized variant --- generator/CMakeLists.txt | 10 ++- generator/generator.cpp | 159 +++++++++++++++++++++++++++++++------------- generator/godot_variant.cpp | 69 +++++++++++++++++++ generator/godot_variant.h | 28 ++++++++ 4 files changed, 217 insertions(+), 49 deletions(-) create mode 100644 generator/godot_variant.cpp create mode 100644 generator/godot_variant.h diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index fb5dc3c..b6d3c42 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -1,11 +1,15 @@ cmake_minimum_required (VERSION 3.1) project (generator) +find_package(PkgConfig) +pkg_check_modules(libzstd libzstd REQUIRED) + add_subdirectory(vendor/fmt) -include_directories(vendor/hkutil vendor/fmt/include) +include_directories(vendor/hkutil vendor/fmt/include ${libzstd_INCLUDE_DIRS}) +link_directories(${libzstd_LIBRARY_DIRS}) -add_executable(generator generator.cpp main.cpp) +add_executable(generator generator.cpp main.cpp godot_variant.cpp) set_property(TARGET generator PROPERTY CXX_STANDARD 17) set_property(TARGET generator PROPERTY CXX_STANDARD_REQUIRED ON) -target_link_libraries(generator fmt) +target_link_libraries(generator fmt ${libzstd_LIBRARIES}) diff --git a/generator/generator.cpp b/generator/generator.cpp index 7263591..7016885 100644 --- a/generator/generator.cpp +++ b/generator/generator.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -10,13 +11,17 @@ #include #include #include +#include #include #include #include #include #include +#include "godot_variant.h" + constexpr int MIN_FREQUENCY = 2000000; +constexpr int kCompressionLevel = 3; namespace { @@ -877,7 +882,7 @@ void generator::run() { } // Orange addition - std::vector orange_addition; + std::vector orange_addition; { hatkirby::progress ppgs("Generating orange addition puzzles...", wanderlust_.size()); @@ -894,8 +899,12 @@ void generator::run() { continue; } if (wanderlust_.count(cipher - cipher2)) { - orange_addition.push_back(fmt::format( - "[{},{},{}]", form_id2, wanderlust_[cipher - cipher2], form_id)); + std::vector addition_entry; + addition_entry.emplace_back(static_cast(form_id2)); + addition_entry.emplace_back( + static_cast(wanderlust_[cipher - cipher2])); + addition_entry.emplace_back(static_cast(form_id)); + orange_addition.emplace_back(addition_entry); } } } @@ -995,83 +1004,141 @@ void generator::run() { std::cout << "Yellow top yellow middle combos: " << combos_[kYellowTop][kYellowMiddle].size() << std::endl; - std::vector form_entry; + // 0: forms + // 1: paintings + // 2: wanderlust + // 3: addition + // 4: walls + // 5: combos + std::vector full_variant; + + std::vector form_entry; form_entry.reserve(forms_.size()); for (const Form& form : forms_) { if (form.puzzles.empty()) { - form_entry.push_back(fmt::format("\"{}\"", form.text)); + form_entry.emplace_back(form.text); } else { - std::vector entry_per_type; + std::map entry_per_type; for (const auto& [puzzle_type, puzzles] : form.puzzles) { - std::vector entry_per_puzzle; + std::vector entry_per_puzzle; for (size_t puzzle : puzzles) { - entry_per_puzzle.push_back(std::to_string(puzzle)); + entry_per_puzzle.emplace_back(puzzle); } - entry_per_type.push_back( - fmt::format("{}:[{}]", static_cast(puzzle_type), - hatkirby::implode(entry_per_puzzle, ","))); + entry_per_type.emplace( + std::piecewise_construct, + std::forward_as_tuple(static_cast(puzzle_type)), + std::forward_as_tuple(entry_per_puzzle)); + } + std::vector pair_entry; + pair_entry.emplace_back(form.text); + if (!entry_per_type.empty()) { + pair_entry.emplace_back(entry_per_type); } - form_entry.push_back(fmt::format("[\"{}\",{{{}}}]", form.text, - hatkirby::implode(entry_per_type, ","))); + form_entry.emplace_back(pair_entry); } } + full_variant.emplace_back(form_entry); - std::ofstream output_file(outputPath_); - output_file << "extends Node\n\nvar forms = [" - << hatkirby::implode(form_entry, ",") << "]" << std::endl; - - std::vector painting_entries; + std::vector painting_entries; { std::list paintings(readFile(datadirPath_ / "paintings.txt")); for (const std::string& line : paintings) { auto parts = hatkirby::split>(line, ","); - painting_entries.push_back( - fmt::format("[\"{}\",\"{}\"]", parts[0], parts[1])); + std::vector painting_entry; + painting_entry.emplace_back(parts[0]); + painting_entry.emplace_back(parts[1]); + painting_entries.emplace_back(painting_entry); } } - output_file << "var paintings = [" << hatkirby::implode(painting_entries, ",") - << "]" << std::endl; + full_variant.emplace_back(painting_entries); - std::vector cipher_lines; + std::vector wanderlust_entries; for (const auto& [cipher, form_id] : wanderlust_) { - cipher_lines.push_back(std::to_string(form_id)); + wanderlust_entries.emplace_back(static_cast(form_id)); } - output_file << "var wanderlust = [" << hatkirby::implode(cipher_lines, ",") - << "]" << std::endl; - output_file << "var addition = [" << hatkirby::implode(orange_addition, ",") - << "]" << std::endl; + full_variant.emplace_back(wanderlust_entries); + + full_variant.emplace_back(orange_addition); - std::vector walls_entries; + std::vector walls_entries; { std::list walls(readFile(datadirPath_ / "walls.txt")); for (const std::string& line : walls) { auto parts = hatkirby::split>(line, ","); - walls_entries.push_back( - fmt::format("[\"{}\",\"{}\"]", parts[0], parts[1])); + std::vector walls_entry; + walls_entry.emplace_back(parts[0]); + walls_entry.emplace_back(parts[1]); + walls_entries.emplace_back(walls_entry); } } - output_file << "var walls_puzzles = [" - << hatkirby::implode(walls_entries, ",") << "]" << std::endl; + full_variant.emplace_back(walls_entries); - std::vector combo_entries; + std::map combo_entries; for (const auto& [left_type, left_join] : combos_) { - std::vector left_entries; + std::map left_entries; for (const auto& [right_type, choices] : left_join) { - std::vector choice_entries; + std::vector choice_entries; for (const auto& [hint1, hint2, answer] : choices) { - choice_entries.push_back( - fmt::format("[{},{},{}]", hint1, hint2, answer)); + std::vector choice_entry; + choice_entry.emplace_back(hint1); + choice_entry.emplace_back(hint2); + choice_entry.emplace_back(answer); + choice_entries.emplace_back(choice_entry); } - left_entries.push_back( - fmt::format("{}:[{}]", static_cast(right_type), - hatkirby::implode(choice_entries, ","))); + left_entries.emplace( + std::piecewise_construct, + std::forward_as_tuple(static_cast(right_type)), + std::forward_as_tuple(choice_entries)); } - combo_entries.push_back(fmt::format("{}:{{{}}}", - static_cast(left_type), - hatkirby::implode(left_entries, ","))); + combo_entries.emplace( + std::piecewise_construct, + std::forward_as_tuple(static_cast(left_type)), + std::forward_as_tuple(left_entries)); + } + full_variant.emplace_back(combo_entries); + + std::ostringstream serialized_variant_buffer; + GodotVariant(full_variant).Serialize(serialized_variant_buffer); + + std::string serialized_variant = serialized_variant_buffer.str(); + std::ostringstream variant_file_buffer; + int32_t variant_len = serialized_variant.size(); + variant_file_buffer.write(reinterpret_cast(&variant_len), 4); + variant_file_buffer.write(serialized_variant.data(), + serialized_variant.size()); + + std::string variant_file = variant_file_buffer.str(); + + size_t max_output_size = ZSTD_compressBound(variant_file.size()); + std::string compressed_variant(max_output_size, 0); + size_t compressed_size = ZSTD_compress( + compressed_variant.data(), max_output_size, variant_file.data(), + variant_file.size(), kCompressionLevel); + + // Write Godot's weird gzip container + { + std::ofstream output_file(outputPath_, std::ios::binary); + + int32_t bytes = 0x46504347; + output_file.write(reinterpret_cast(&bytes), 4); + + bytes = 0x00000002; // ZSTD + output_file.write(reinterpret_cast(&bytes), 4); + + bytes = variant_file.size() + 1; + output_file.write(reinterpret_cast(&bytes), 4); + + bytes = variant_file.size(); + output_file.write(reinterpret_cast(&bytes), 4); + + bytes = compressed_size; + output_file.write(reinterpret_cast(&bytes), 4); + + output_file.write(compressed_variant.data(), compressed_size); + + bytes = 0x46504347; + output_file.write(reinterpret_cast(&bytes), 4); } - output_file << "var combos = {" << hatkirby::implode(combo_entries, ",") - << "}" << std::endl; } size_t generator::LookupOrCreatePronunciation(const std::string& phonemes) { diff --git a/generator/godot_variant.cpp b/generator/godot_variant.cpp new file mode 100644 index 0000000..dd1ad33 --- /dev/null +++ b/generator/godot_variant.cpp @@ -0,0 +1,69 @@ +#include "godot_variant.h" + +#include + +bool GodotVariant::operator<(const GodotVariant& rhs) const { + if (value.index() != rhs.value.index()) { + return value.index() < rhs.value.index(); + } + + if (const int32_t* pval = std::get_if(&value)) { + return *pval < std::get(rhs.value); + } else if (const std::string* pval = std::get_if(&value)) { + return *pval < std::get(rhs.value); + } else if (const std::map* pval = + std::get_if>(&value)) { + return *pval < std::get>(rhs.value); + } else if (const std::vector* pval = + std::get_if>(&value)) { + return *pval < std::get>(rhs.value); + } + + throw std::invalid_argument("Incomparable variants."); +} + +void GodotVariant::Serialize(std::basic_ostream& output) const { + if (const int32_t* pval = std::get_if(&value)) { + uint32_t typebits = 2; + output.write(reinterpret_cast(&typebits), 4); + output.write(reinterpret_cast(pval), 4); + } else if (const std::string* pval = std::get_if(&value)) { + uint32_t typebits = 4; + output.write(reinterpret_cast(&typebits), 4); + + uint32_t lengthbits = pval->length(); + output.write(reinterpret_cast(&lengthbits), 4); + + output.write(pval->data(), pval->length()); + + if (pval->length() % 4 != 0) { + int padding = 4 - (pval->length() % 4); + for (int i = 0; i < padding; i++) { + output.put(0); + } + } + } else if (const std::map* pval = + std::get_if>(&value)) { + uint32_t typebits = 18; + output.write(reinterpret_cast(&typebits), 4); + + uint32_t lengthbits = pval->size() & 0x7fffffff; + output.write(reinterpret_cast(&lengthbits), 4); + + for (const auto& mapping : *pval) { + mapping.first.Serialize(output); + mapping.second.Serialize(output); + } + } else if (const std::vector* pval = + std::get_if>(&value)) { + uint32_t typebits = 19; + output.write(reinterpret_cast(&typebits), 4); + + uint32_t lengthbits = pval->size() & 0x7fffffff; + output.write(reinterpret_cast(&lengthbits), 4); + + for (const auto& entry : *pval) { + entry.Serialize(output); + } + } +} diff --git a/generator/godot_variant.h b/generator/godot_variant.h new file mode 100644 index 0000000..6f2651a --- /dev/null +++ b/generator/godot_variant.h @@ -0,0 +1,28 @@ +#ifndef GODOT_VARIANT_H_84682779 +#define GODOT_VARIANT_H_84682779 + +#include +#include +#include +#include +#include +#include + +struct GodotVariant { + using variant_type = std::variant, + std::vector >; + + variant_type value; + + GodotVariant(int32_t v) : value(v) {} + GodotVariant(std::string v) : value(std::move(v)) {} + GodotVariant(std::map v) : value(std::move(v)) {} + GodotVariant(std::vector v) : value(std::move(v)) {} + + bool operator<(const GodotVariant& rhs) const; + + void Serialize(std::basic_ostream& output) const; +}; + +#endif /* end of include guard: GODOT_VARIANT_H_84682779 */ -- cgit 1.4.1