diff --git a/README.md b/README.md index aa40546..aebce25 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ from the module's release listing below. | Module | Formats | Release | ChangeLog | |--------|---------|---------|-----------| -| Ren'Py | `.rpa` (RPA-2.0, RPA-3.0) | [renpy](https://github.com/refaim/unpackrat/releases?q=renpy) | [RU](src/modules/renpy/ChangeLog.ru.txt) [EN](src/modules/renpy/ChangeLog.en.txt) | +| Ren'Py | `.rpa` (RPA-1.0 with adjacent `.rpi`, RPA-2.0, RPA-3.0) | [renpy](https://github.com/refaim/unpackrat/releases?q=renpy) | [RU](src/modules/renpy/ChangeLog.ru.txt) [EN](src/modules/renpy/ChangeLog.en.txt) | | RPG Maker | `.rgssad` (XP, RGSS1), `.rgss2a` (VX, RGSS2), `.rgss3a` (VX Ace, RGSS3) | [rpgmaker](https://github.com/refaim/unpackrat/releases?q=rpgmaker) | [RU](src/modules/rpgmaker/ChangeLog.ru.txt) [EN](src/modules/rpgmaker/ChangeLog.en.txt) | | Zanzarah | `.pak` | [zanzarah 2.3.0](https://github.com/refaim/unpackrat/releases?q=zanzarah) | [RU](src/modules/zanzarah/ChangeLog.ru.txt) [EN](src/modules/zanzarah/ChangeLog.en.txt) | | Godot | `.pck` | [godot 1.2.0](https://github.com/refaim/unpackrat/releases?q=godot) | [RU](src/modules/godot/ChangeLog.ru.txt) [EN](src/modules/godot/ChangeLog.en.txt) | diff --git a/build/projects/fuzz-renpy.vcxproj b/build/projects/fuzz-renpy.vcxproj index ebb7ac4..383e142 100644 --- a/build/projects/fuzz-renpy.vcxproj +++ b/build/projects/fuzz-renpy.vcxproj @@ -30,12 +30,14 @@ + + diff --git a/build/projects/renpy.vcxproj b/build/projects/renpy.vcxproj index 25c5608..d965e28 100644 --- a/build/projects/renpy.vcxproj +++ b/build/projects/renpy.vcxproj @@ -34,6 +34,7 @@ + @@ -42,6 +43,7 @@ + diff --git a/build/projects/tests.vcxproj b/build/projects/tests.vcxproj index c058368..8323b9f 100644 --- a/build/projects/tests.vcxproj +++ b/build/projects/tests.vcxproj @@ -102,6 +102,8 @@ $(IntDir)pickle_unit.obj + + $(IntDir)bounded_stream_unit.obj diff --git a/build/tests/test_supported_architecture_docs.py b/build/tests/test_supported_architecture_docs.py index c43aec5..0d70289 100644 --- a/build/tests/test_supported_architecture_docs.py +++ b/build/tests/test_supported_architecture_docs.py @@ -26,6 +26,12 @@ DOCUMENTS = (README, AGENTS, BUILD_SYSTEM, METHODOLOGY) MODULES = ("renpy", "rpgmaker", "zanzarah", "godot") +CONSOLIDATED_RELEASES = { + "renpy": "3.2.0", + "rpgmaker": "1.3.0", + "zanzarah": "2.3.0", + "godot": "1.2.0", +} LANGUAGES = ("en", "ru") @@ -177,6 +183,52 @@ def test_methodology_describes_the_real_binary_gate_without_removed_checks(self) self.assertIn("forbidden imports", docs) self.assertIn("BinSkim", docs) + def test_future_changelog_release_keeps_the_historical_windows_announcement(self) -> None: + for language, header, announcement in ( + ("en", "Version", "+ Added support for Windows XP–11."), + ("ru", "Версия", "+ Добавлена поддержка Windows XP–11."), + ): + with self.subTest(language=language): + changelog = ( + f"{header} 99.0.0\n-------------\n + Added a future feature.\n\n" + f"{header} 3.2.0\n-------------\n {announcement}\n\n" + f"{header} 1.0.0\n-------------\n + Original release.\n" + ) + case = SupportedArchitectureDocumentTests( + "test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history" + ) + with ( + mock.patch(__name__ + ".MODULES", ("renpy",)), + mock.patch(__name__ + ".LANGUAGES", (language,)), + mock.patch(__name__ + ".read", return_value=changelog), + mock.patch.object(Path, "read_text", return_value="99.0.0"), + ): + case.test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history() + + def test_latest_or_other_release_cannot_replace_the_consolidated_windows_announcement(self) -> None: + for language, header, announcement in ( + ("en", "Version", "+ Added support for Windows XP–11."), + ("ru", "Версия", "+ Добавлена поддержка Windows XP–11."), + ): + with self.subTest(language=language): + changelog = ( + f"{header} 99.0.0\n-------------\n {announcement}\n\n" + f"{header} 3.2.0\n-------------\n + A different historical feature.\n\n" + f"{header} 3.1.0\n-------------\n {announcement}\n\n" + f"{header} 1.0.0\n-------------\n + Original release.\n" + ) + case = SupportedArchitectureDocumentTests( + "test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history" + ) + with ( + mock.patch(__name__ + ".MODULES", ("renpy",)), + mock.patch(__name__ + ".LANGUAGES", (language,)), + mock.patch(__name__ + ".read", return_value=changelog), + mock.patch.object(Path, "read_text", return_value="99.0.0"), + self.assertRaisesRegex(AssertionError, "XP"), + ): + case.test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history() + def test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history(self) -> None: headers = {"en": "Version", "ru": "\u0412\u0435\u0440\u0441\u0438\u044f"} for module in MODULES: @@ -203,8 +255,19 @@ def test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history(self) line for line in body[:end] if line.lstrip().startswith(("*", "+")) ] self.assertGreaterEqual(len(entries), 1) - # Every consolidated current release announces Windows XP..11 support; - # a historical announcement alone does not satisfy the release contract. + # The Windows support announcement belongs to its original consolidated + # release. New releases keep that exact historical section without + # repeating the announcement as if Windows support were newly added. + consolidated = f"{headers[language]} {CONSOLIDATED_RELEASES[module]}" + self.assertIn(consolidated, lines) + body = lines[lines.index(consolidated) + 2:] + end = next( + (index for index, line in enumerate(body) if line.startswith(headings)), + len(body), + ) + entries = [ + line for line in body[:end] if line.lstrip().startswith(("*", "+")) + ] announcement = { "en": "+ Added support for Windows XP–11.", "ru": "+ Добавлена поддержка Windows XP–11.", diff --git a/src/modules/renpy/ChangeLog.en.txt b/src/modules/renpy/ChangeLog.en.txt index 648dfe4..f8aa4f8 100644 --- a/src/modules/renpy/ChangeLog.en.txt +++ b/src/modules/renpy/ChangeLog.en.txt @@ -1,3 +1,9 @@ +Version 3.3.0 +------------- + + Added support for RPA 1.0 archives with adjacent .rpi indexes, including files stored in multiple fragments. + * Validate legacy index paths, fragment bounds and metadata limits before opening an archive. + * Limit object and string copying while parsing Pickle indexes to reject memo amplification safely. + Version 3.2.0 ------------- + Added support for Windows XP–11. diff --git a/src/modules/renpy/ChangeLog.ru.txt b/src/modules/renpy/ChangeLog.ru.txt index d4883c4..eaecfcc 100644 --- a/src/modules/renpy/ChangeLog.ru.txt +++ b/src/modules/renpy/ChangeLog.ru.txt @@ -1,3 +1,9 @@ +Версия 3.3.0 +------------- + + Добавлена поддержка архивов RPA 1.0 с соседним индексом .rpi, включая файлы из нескольких фрагментов. + * До открытия архива проверяются пути, границы фрагментов и ограничения размера старого индекса. + * Ограничено копирование объектов и строк в индексах Pickle для защиты от раздувания через memo. + Версия 3.2.0 ------------- + Добавлена поддержка Windows XP–11. diff --git a/src/modules/renpy/VERSION b/src/modules/renpy/VERSION index 944880f..15a2799 100644 --- a/src/modules/renpy/VERSION +++ b/src/modules/renpy/VERSION @@ -1 +1 @@ -3.2.0 +3.3.0 diff --git a/src/modules/renpy/pickle.cpp b/src/modules/renpy/pickle.cpp index e471fc8..39d8a8f 100644 --- a/src/modules/renpy/pickle.cpp +++ b/src/modules/renpy/pickle.cpp @@ -64,49 +64,78 @@ namespace pickle namespace { - list clone_list(const list &source) + void consume_node(limits &remaining) + { + if (remaining.max_nodes == 0) { + throw std::runtime_error("Pickle node budget exceeded"); + } + --remaining.max_nodes; + } + + void consume_bytes(limits &remaining, const std::size_t size) + { + if (size > remaining.max_owned_bytes) { + throw std::runtime_error("Pickle owned byte budget exceeded"); + } + remaining.max_owned_bytes -= size; + } + + value_ptr clone_value(const value &source, limits &remaining); + + list clone_list(const list &source, limits &remaining) { list result; - result.reserve(source.size()); - std::ranges::transform(source, std::back_inserter(result), [](const auto &item) { return clone(*item); }); + result.reserve(std::min(source.size(), remaining.max_nodes)); + std::ranges::transform(source, std::back_inserter(result), + [&remaining](const auto &item) { return clone_value(*item, remaining); }); return result; } - dict clone_dict(const dict &source) + dict clone_dict(const dict &source, limits &remaining) { dict result; - result.reserve(source.size()); + result.reserve(std::min(source.size(), remaining.max_nodes)); for (const auto &[key, item] : source) { - result.emplace(key, clone(*item)); + consume_bytes(remaining, key.size()); + result.emplace(key, clone_value(*item, remaining)); } return result; } + + value_ptr clone_value(const value &source, limits &remaining) + { + consume_node(remaining); + switch (source.get_type()) { + case value::type::none: + return value::none(); + case value::type::bool_: + return value::boolean(source.as_bool()); + case value::type::int64: + return value::int64(source.as_int64()); + case value::type::float64: + return value::float64(source.as_float64()); + case value::type::bytes: + consume_bytes(remaining, source.as_string().size()); + return value::bytes(source.as_string()); + case value::type::string: + consume_bytes(remaining, source.as_string().size()); + return value::string(source.as_string()); + case value::type::list: + return value::list(clone_list(source.as_list(), remaining)); + case value::type::dict: + return value::dict(clone_dict(source.as_dict(), remaining)); + case value::type::tuple: + return value::tuple(clone_list(source.as_tuple(), remaining)); + default: + throw std::runtime_error("Unsupported pickle value type"); + } + } } // namespace - value_ptr clone(const value &source) + value_ptr clone(const value &source, const limits &bounds) { - switch (source.get_type()) { - case value::type::none: - return value::none(); - case value::type::bool_: - return value::boolean(source.as_bool()); - case value::type::int64: - return value::int64(source.as_int64()); - case value::type::float64: - return value::float64(source.as_float64()); - case value::type::bytes: - return value::bytes(source.as_string()); - case value::type::string: - return value::string(source.as_string()); - case value::type::list: - return value::list(clone_list(source.as_list())); - case value::type::dict: - return value::dict(clone_dict(source.as_dict())); - case value::type::tuple: - return value::tuple(clone_list(source.as_tuple())); - default: - throw std::runtime_error("Unsupported pickle value type"); - } + auto remaining = bounds; + return clone_value(source, remaining); } // Pickle opcodes (subset needed for basic functionality) @@ -206,6 +235,7 @@ namespace pickle if (pos_ + length > data_.size()) { throw std::runtime_error("Unexpected end of pickle data"); } + consume_bytes(remaining_, length); std::string result(reinterpret_cast(data_.data() + pos_), length); pos_ += length; return result; @@ -220,6 +250,7 @@ namespace pickle if (pos_ >= data_.size()) { throw std::runtime_error("Unexpected end of pickle data"); } + consume_bytes(remaining_, pos_ - start); std::string result(reinterpret_cast(data_.data() + start), pos_ - start); pos_++; // skip newline return result; @@ -264,26 +295,32 @@ namespace pickle return std::move(stack_[0]); case opcodes::NONE: + consume_node(remaining_); stack_.push_back(value::none()); break; case opcodes::NEWTRUE: + consume_node(remaining_); stack_.push_back(value::boolean(true)); break; case opcodes::NEWFALSE: + consume_node(remaining_); stack_.push_back(value::boolean(false)); break; case opcodes::BININT: + consume_node(remaining_); stack_.push_back(value::int64(static_cast(read_uint32_le()))); break; case opcodes::BININT1: + consume_node(remaining_); stack_.push_back(value::int64(read_byte())); break; case opcodes::BININT2: + consume_node(remaining_); stack_.push_back(value::int64(read_uint16_le())); break; @@ -296,6 +333,7 @@ namespace pickle int_str.pop_back(); // Remove trailing L } int64_t val = std::stoll(int_str); + consume_node(remaining_); stack_.push_back(value::int64(val)); break; } @@ -304,47 +342,55 @@ namespace pickle const uint64_t bits = std::byteswap(read_uint64_le()); double val; std::memcpy(&val, &bits, sizeof(double)); + consume_node(remaining_); stack_.push_back(value::float64(val)); break; } case opcodes::SHORT_BINSTRING: { uint8_t length = read_byte(); + consume_node(remaining_); stack_.push_back(value::string(read_string(length))); break; } case opcodes::BINSTRING: { uint32_t length = read_uint32_le(); + consume_node(remaining_); stack_.push_back(value::string(read_string(length))); break; } case opcodes::SHORT_BINUNICODE: { uint8_t length = read_byte(); + consume_node(remaining_); stack_.push_back(value::string(read_string(length))); break; } case opcodes::BINUNICODE: { uint32_t length = read_uint32_le(); + consume_node(remaining_); stack_.push_back(value::string(read_string(length))); break; } case opcodes::SHORT_BINBYTES: { uint8_t length = read_byte(); + consume_node(remaining_); stack_.push_back(value::bytes(read_string(length))); break; } case opcodes::BINBYTES: { uint32_t length = read_uint32_le(); + consume_node(remaining_); stack_.push_back(value::bytes(read_string(length))); break; } case opcodes::EMPTY_LIST: + consume_node(remaining_); stack_.push_back(value::list({})); break; @@ -377,16 +423,19 @@ namespace pickle case opcodes::LIST: { auto items = pop_to_mark(); + consume_node(remaining_); stack_.push_back(value::list(std::move(items))); break; } case opcodes::EMPTY_TUPLE: + consume_node(remaining_); stack_.push_back(value::tuple({})); break; case opcodes::TUPLE: { auto items = pop_to_mark(); + consume_node(remaining_); stack_.push_back(value::tuple(std::move(items))); break; } @@ -399,6 +448,7 @@ namespace pickle stack_.pop_back(); list tuple_items; tuple_items.push_back(std::move(item)); + consume_node(remaining_); stack_.push_back(value::tuple(std::move(tuple_items))); break; } @@ -414,6 +464,7 @@ namespace pickle list tuple_items; tuple_items.push_back(std::move(item1)); tuple_items.push_back(std::move(item2)); + consume_node(remaining_); stack_.push_back(value::tuple(std::move(tuple_items))); break; } @@ -432,11 +483,13 @@ namespace pickle tuple_items.push_back(std::move(item1)); tuple_items.push_back(std::move(item2)); tuple_items.push_back(std::move(item3)); + consume_node(remaining_); stack_.push_back(value::tuple(std::move(tuple_items))); break; } case opcodes::EMPTY_DICT: + consume_node(remaining_); stack_.push_back(value::dict({})); break; @@ -450,9 +503,11 @@ namespace pickle if (items[i]->get_type() != value::type::string) { throw std::runtime_error("Dict key must be string"); } + consume_bytes(remaining_, items[i]->as_string().size()); std::string key = items[i]->as_string(); dict_data[std::move(key)] = std::move(items[i + 1]); } + consume_node(remaining_); stack_.push_back(value::dict(std::move(dict_data))); break; } @@ -474,6 +529,7 @@ namespace pickle throw std::runtime_error("Dict key must be string"); } + consume_bytes(remaining_, key->as_string().size()); dict_val->set(key->as_string(), std::move(val)); break; } @@ -498,6 +554,7 @@ namespace pickle value::checked_child_depth(items[i + 1]); } for (size_t i = 0; i < items.size(); i += 2) { + consume_bytes(remaining_, items[i]->as_string().size()); std::string key = items[i]->as_string(); dict_val->set(std::move(key), std::move(items[i + 1])); } @@ -509,7 +566,7 @@ namespace pickle if (stack_.empty()) { throw std::runtime_error("No item on stack for BINPUT"); } - memo_[memo_id] = clone(*stack_.back()); + memo_[memo_id] = clone_value(*stack_.back(), remaining_); break; } @@ -518,7 +575,7 @@ namespace pickle if (stack_.empty()) { throw std::runtime_error("No item on stack for LONG_BINPUT"); } - memo_[memo_id] = clone(*stack_.back()); + memo_[memo_id] = clone_value(*stack_.back(), remaining_); break; } @@ -528,7 +585,7 @@ namespace pickle if (it == memo_.end()) { throw std::runtime_error("Memo key not found"); } - stack_.push_back(clone(*it->second)); + stack_.push_back(clone_value(*it->second, remaining_)); break; } @@ -538,7 +595,7 @@ namespace pickle if (it == memo_.end()) { throw std::runtime_error("Memo key not found"); } - stack_.push_back(clone(*it->second)); + stack_.push_back(clone_value(*it->second, remaining_)); break; } @@ -557,6 +614,7 @@ namespace pickle case opcodes::LONG1: { uint8_t length = read_byte(); if (length == 0) { + consume_node(remaining_); stack_.push_back(value::int64(0)); } else { if (length > sizeof(std::int64_t)) { @@ -574,6 +632,7 @@ namespace pickle bits |= ~std::uint64_t{0} << length * 8U; } + consume_node(remaining_); stack_.push_back(value::int64(std::bit_cast(bits))); } break; @@ -584,7 +643,7 @@ namespace pickle throw std::runtime_error("No item on stack for MEMOIZE"); } const auto memo_id = static_cast(memo_.size()); - memo_[memo_id] = clone(*stack_.back()); + memo_[memo_id] = clone_value(*stack_.back(), remaining_); break; } @@ -605,15 +664,15 @@ namespace pickle throw std::runtime_error("Unexpected end of pickle data"); } - value_ptr loads(const std::span data) + value_ptr loads(const std::span data, const limits &bounds) { - parser p(data); + parser p(data, bounds); return p.parse(); } - value_ptr loads(const std::string &data) + value_ptr loads(const std::string &data, const limits &bounds) { const auto byte_span = std::span(reinterpret_cast(data.data()), data.size()); - return loads(byte_span); + return loads(byte_span, bounds); } } // namespace pickle diff --git a/src/modules/renpy/pickle.h b/src/modules/renpy/pickle.h index b95ac89..dd5d7d3 100644 --- a/src/modules/renpy/pickle.h +++ b/src/modules/renpy/pickle.h @@ -19,6 +19,12 @@ namespace pickle using list = std::vector; using dict = std::unordered_map; + struct limits final + { + std::size_t max_nodes = 1'000'000; + std::size_t max_owned_bytes = std::size_t{64} * 1024 * 1024; + }; + class value { public: @@ -197,7 +203,7 @@ namespace pickle } }; - value_ptr clone(const value &source); + value_ptr clone(const value &source, const limits &bounds = {}); class parser { @@ -207,6 +213,9 @@ namespace pickle std::vector stack_; std::vector mark_stack_; std::unordered_map memo_; + // Monotonic work/ownership allowances cover original values, every memo clone, + // and copied strings and dictionary keys, even after a memo slot is replaced. + limits remaining_; uint8_t read_byte(); @@ -227,15 +236,16 @@ namespace pickle value_ptr parse_value(); public: - explicit parser(const std::span data) : data_(data) + explicit parser(const std::span data, const limits &bounds = {}) + : data_(data), remaining_(bounds) { } value_ptr parse(); }; - value_ptr loads(std::span data); + value_ptr loads(std::span data, const limits &bounds = {}); - value_ptr loads(const std::string &data); + value_ptr loads(const std::string &data, const limits &bounds = {}); } // namespace pickle #endif diff --git a/src/modules/renpy/renpy.cpp b/src/modules/renpy/renpy.cpp index 3a49e0e..acb76fe 100644 --- a/src/modules/renpy/renpy.cpp +++ b/src/modules/renpy/renpy.cpp @@ -2,7 +2,12 @@ #include "../../core/io/bounded_stream.h" #include "../extractor.h" #include "pickle.h" +#include "rpa1_index.h" +#include +#include +#include +#include #include #include #include @@ -12,6 +17,56 @@ namespace extractor { + namespace + { + constexpr std::string_view rpa_signature = "RPA-"; + + struct rpa1_storage final : storage_state + { + std::vector entries; + }; + + bool valid_modern_index(extractor &parser, std::istream &stream) + { + try { + static_cast(parser.list_files(stream)); + return true; + } catch (const std::runtime_error &) { + return false; + } catch (const std::out_of_range &) { + return false; + } catch (const std::invalid_argument &) { + return false; + } + } + + void extract_fragments(std::istream &source, std::ostream &output, + const std::vector &fragments, + const std::function &report_progress) + { + constexpr std::int64_t chunk_bytes = 128LL * 1024; + std::vector buffer(static_cast(chunk_bytes)); + for (const auto &fragment : fragments) { + auto position = fragment.offset; + auto remaining = fragment.size; + while (remaining > 0) { + const auto count = std::min(remaining, chunk_bytes); + try { + source.clear(); + source.seekg(position); + source.read(buffer.data(), static_cast(count)); + } catch (const std::ios_base::failure &) { + throw read_error(); + } + output.write(buffer.data(), static_cast(count)); + report_progress(count); + position += count; + remaining -= count; + } + } + } + } // namespace + void configure([[maybe_unused]] const wchar_t *settings) noexcept { } @@ -30,18 +85,55 @@ namespace extractor std::vector extractor::get_signature() { - const std::string str = "RPA-"; - std::vector signature(str.size()); - std::memcpy(signature.data(), str.data(), str.size()); - return signature; + // RPA 1.0 data has no signature. Recognition below requires its exact adjacent index. + return {}; } - archive_info extractor::get_archive_info( - [[maybe_unused]] std::istream &stream, const std::span &data, - [[maybe_unused]] const std::filesystem::path &path, - [[maybe_unused]] const std::string_view password) // NOLINT(*-convert-member-functions-to-static) + archive_info extractor::get_archive_info(std::istream &stream, const std::span &data, + const std::filesystem::path &path, + [[maybe_unused]] const std::string_view password) { - static_cast(data); + unpackrat::io::bounded_stream input(stream); + std::array prefix{}; + const auto prefix_size = + static_cast(std::min(input.size(), static_cast(rpa_signature.size()))); + input.seek_absolute(0); + input.read_exact(prefix.data(), prefix_size); + const bool has_modern_signature = std::string_view(prefix.data(), prefix_size) == rpa_signature; + auto extension = path.extension().wstring(); + std::ranges::transform(extension, extension.begin(), [](const wchar_t value) { return std::towlower(value); }); + auto index_path = path; + index_path.replace_extension(L".rpi"); + + // A headerless RPA1 body may contain any bytes, including a modern header. If both + // representations are present, a valid modern archive wins; otherwise try its sidecar. + if (has_modern_signature && + (extension != L".rpa" || !std::filesystem::exists(index_path) || valid_modern_index(*this, stream))) { + if (!data.empty() && (data.size() < rpa_signature.size() || + std::memcmp(data.data(), rpa_signature.data(), rpa_signature.size()) != 0)) { + throw std::runtime_error("Signature mismatch"); + } + return archive_info{L"RenPy", L"", L""}; + } + + if (extension != L".rpa") { + throw std::runtime_error("RPA 1.0 requires a .rpa data file"); + } + std::ifstream index(index_path, std::ios::binary); + if (!index.is_open()) { + throw std::runtime_error("RPA 1.0 index is missing or unreadable"); + } + unpackrat::io::bounded_stream index_input(index); + const auto index_size = static_cast(index_input.size()); + if (index_size > unpackrat::renpy::rpa1_limits{}.max_compressed_index_bytes) { + throw std::runtime_error("RPA compressed index exceeds the metadata budget"); + } + std::vector compressed_index(static_cast(index_size)); + index_input.seek_absolute(0); + index_input.read_exact(reinterpret_cast(compressed_index.data()), compressed_index.size()); + auto state = std::make_unique(); + state->entries = unpackrat::renpy::parse_rpa1_index(compressed_index, input.size()); + state_ = std::move(state); return archive_info{L"RenPy", L"", L""}; } @@ -64,7 +156,7 @@ namespace extractor unpackrat::io::bounded_stream &stream) { std::string version_check(3, '\0'); - stream.seek_absolute(static_cast(extractor::get_signature().size())); + stream.seek_absolute(static_cast(rpa_signature.size())); stream.read_exact(version_check.data(), version_check.size()); if (version_check == "2.0") { @@ -87,9 +179,26 @@ namespace extractor throw std::runtime_error("Unsupported RPA version"); } - std::vector> extractor::list_files( - std::istream &stream) // NOLINT(*-convert-member-functions-to-static) + std::vector> extractor::list_files(std::istream &stream) { + if (state_) { + const auto &state = static_cast(*state_); + std::vector> files; + files.reserve(state.entries.size()); + for (const auto &entry : state.entries) { + auto item = std::make_unique(); + item->path = entry.path; + item->compressed_body_size_in_bytes = entry.size; + item->uncompressed_body_size_in_bytes = entry.size; + item->extract = [fragments = entry.fragments](std::istream &source, std::ostream &output, + const std::function &report_progress) { + extract_fragments(source, output, fragments, report_progress); + }; + files.push_back(std::move(item)); + } + return files; + } + unpackrat::io::bounded_stream input(stream); const auto [index_offset, decoder] = parse_header(input); const auto archive_size = input.size(); diff --git a/src/modules/renpy/rpa1_index.cpp b/src/modules/renpy/rpa1_index.cpp new file mode 100644 index 0000000..de63ff3 --- /dev/null +++ b/src/modules/renpy/rpa1_index.cpp @@ -0,0 +1,110 @@ +#include "rpa1_index.h" +#include "../../core/compression/zlib_codec.h" +#include "pickle.h" + +#include +#include +#include +#include + +namespace unpackrat::renpy +{ + namespace + { + pickle::value_ptr load_index(const std::span data) + { + try { + return pickle::loads(data); + } catch (const std::invalid_argument &) { + throw std::runtime_error("RPA1 index contains an invalid integer"); + } catch (const std::out_of_range &) { + throw std::runtime_error("RPA1 index contains an integer outside int64"); + } + } + + void validate_path(const std::string_view path, const std::size_t max_path_bytes) + { + if (path.empty()) { + throw std::runtime_error("RPA1 entry path is empty"); + } + if (path.size() > max_path_bytes) { + throw std::runtime_error("RPA1 entry path exceeds the path budget"); + } + if (path.find_first_of(std::string_view("\0:", 2)) != std::string_view::npos) { + throw std::runtime_error("RPA1 entry path contains a forbidden character"); + } + if (path.find_first_of("/\\") == 0) { + throw std::runtime_error("RPA1 entry path is absolute"); + } + + std::size_t begin = 0; + for (;;) { + const auto end = path.find_first_of("/\\", begin); + const auto segment = path.substr(begin, end == std::string_view::npos ? end : end - begin); + if (segment == "..") { + throw std::runtime_error("RPA1 entry path contains parent traversal"); + } + if (end == std::string_view::npos) { + return; + } + begin = end + 1; + } + } + } // namespace + + std::vector parse_rpa1_index(const std::span compressed_index, + const std::int64_t archive_size, const rpa1_limits &limits) + { + if (archive_size < 0) { + throw std::runtime_error("RPA1 data file size is negative"); + } + if (compressed_index.size() > limits.max_compressed_index_bytes) { + throw std::runtime_error("RPA1 compressed index exceeds the metadata budget"); + } + const auto decompressed = compression::decompress_zlib(compressed_index, limits.max_decompressed_index_bytes); + const auto root = load_index(decompressed); + const auto &dict = root->as_dict(); + if (dict.size() > limits.max_entries) { + throw std::runtime_error("RPA1 index exceeds the entry budget"); + } + + std::vector result; + result.reserve(dict.size()); + std::size_t fragment_count = 0; + for (const auto &[path, value] : dict) { + validate_path(path, limits.max_path_bytes); + const auto &fragments = value->as_list(); + if (fragments.empty()) { + throw std::runtime_error("RPA1 entry has no fragments"); + } + if (fragments.size() > limits.max_fragments - fragment_count) { + throw std::runtime_error("RPA1 index exceeds the fragment budget"); + } + fragment_count += fragments.size(); + + rpa1_entry entry{path, {}, 0}; + entry.fragments.reserve(fragments.size()); + for (const auto &fragment : fragments) { + const auto &tuple = fragment->as_tuple(); + if (tuple.size() != 2) { + throw std::runtime_error("RPA1 fragment must contain exactly two integers"); + } + const auto offset = tuple[0]->as_int64(); + const auto size = tuple[1]->as_int64(); + if (offset < 0 || size < 0) { + throw std::runtime_error("RPA1 fragment has a negative extent"); + } + if (offset > archive_size || size > archive_size - offset) { + throw std::runtime_error("RPA1 fragment exceeds the data file"); + } + if (size > std::numeric_limits::max() - entry.size) { + throw std::runtime_error("RPA1 entry size overflows int64"); + } + entry.size += size; + entry.fragments.push_back({offset, size}); + } + result.push_back(std::move(entry)); + } + return result; + } +} // namespace unpackrat::renpy diff --git a/src/modules/renpy/rpa1_index.h b/src/modules/renpy/rpa1_index.h new file mode 100644 index 0000000..188287c --- /dev/null +++ b/src/modules/renpy/rpa1_index.h @@ -0,0 +1,43 @@ +#ifndef UNPACKRAT_MODULES_RENPY_RPA1_INDEX_H +#define UNPACKRAT_MODULES_RENPY_RPA1_INDEX_H + +#include "../../core/archive_limits.h" + +#include +#include +#include +#include +#include + +namespace unpackrat::renpy +{ + struct rpa1_fragment final + { + std::int64_t offset; + std::int64_t size; + }; + + struct rpa1_entry final + { + std::string path; + std::vector fragments; + std::int64_t size = 0; + }; + + struct rpa1_limits final + { + std::size_t max_compressed_index_bytes = std::size_t{64} * 1024 * 1024; + std::size_t max_decompressed_index_bytes = std::size_t{64} * 1024 * 1024; + std::size_t max_entries = archive_limits::max_entry_count; + std::size_t max_fragments = 1'000'000; + std::size_t max_path_bytes = archive_limits::max_path_bytes; + }; + + // RPA1 stores its zlib-compressed Pickle index separately from the headerless data file. + // Each path maps to an ordered, nonempty list of (offset, size) fragments. The caller + // supplies the data file's authoritative size; all returned extents fit that file. + [[nodiscard]] std::vector parse_rpa1_index(std::span compressed_index, + std::int64_t archive_size, const rpa1_limits &limits = {}); +} // namespace unpackrat::renpy + +#endif diff --git a/src/tests/data/renpy/README.md b/src/tests/data/renpy/README.md index 0adbcbc..a5f5eec 100644 --- a/src/tests/data/renpy/README.md +++ b/src/tests/data/renpy/README.md @@ -1,4 +1,6 @@ -# Oversized Ren'Py index fixture +# Ren'Py archive fixtures + +## Oversized index fixture `expanded-index-over-budget.rpa` is a 65,271-byte RPA-2.0 archive. Its index begins at offset 32 and is a zlib stream that expands to exactly 67,108,865 `N` bytes: @@ -21,3 +23,36 @@ header = b"RPA-2.0 " + format(32, "016x").encode("ascii") + b"\n" archive = header.ljust(32, b"\0") + zlib.compress(b"N" * expanded_size, 9) Path("expanded-index-over-budget.rpa").write_bytes(archive) ``` + +## RPA-1.0 split-file pair + +`rpa1-split.rpa` and `rpa1-split.rpi` are authored test data, unrelated to any +game assets. The 29-byte data file contains padding and three fragments of +`dir/hello.txt`; the index lists the fragments in logical order, deliberately +different from their physical order. Extracting that entry must produce exactly +the 18 ASCII bytes `first:second:third`. The second entry, `empty.txt`, has a +zero-length range at the end of the data file and must extract as an empty file. +The `.rpa` file is the archive entry point; its neighboring `.rpi` file is a +required index sidecar and must not open as an archive by itself. + +The 85-byte index is a zlib-compressed protocol-2 Pickle dictionary. Regenerate +both files from this directory with Python's standard library: + +```python +from pathlib import Path +import pickle +import zlib + +data = b"PAD!third\xfffirst:\x00\x01second:TAIL" +index = { + "dir/hello.txt": [(10, 6), (18, 7), (4, 5)], + "empty.txt": [(len(data), 0)], +} +Path("rpa1-split.rpa").write_bytes(data) +Path("rpa1-split.rpi").write_bytes(zlib.compress(pickle.dumps(index, protocol=2), 9)) +``` + +SHA-256 values of the checked-in files: + +- `rpa1-split.rpa`: `cc75c66d110b50aa52eed163d72bae23953e2ac0a81a0856a5a0f06a2931ea68`. +- `rpa1-split.rpi`: `03fd8fb47ed04b503141d4520a564c269eb1ff1fcedf1b88980ff6b94c172bef`. diff --git a/src/tests/data/renpy/rpa1-split.rpa b/src/tests/data/renpy/rpa1-split.rpa new file mode 100644 index 0000000..588dc5c Binary files /dev/null and b/src/tests/data/renpy/rpa1-split.rpa differ diff --git a/src/tests/data/renpy/rpa1-split.rpi b/src/tests/data/renpy/rpa1-split.rpi new file mode 100644 index 0000000..820081f Binary files /dev/null and b/src/tests/data/renpy/rpa1-split.rpi differ diff --git a/src/tests/framework/observer.cpp b/src/tests/framework/observer.cpp index 1eb188c..a221914 100644 --- a/src/tests/framework/observer.cpp +++ b/src/tests/framework/observer.cpp @@ -3,9 +3,11 @@ #include "../../core/archive_limits.h" #include "../../core/godot/pck.h" #include "../support/archive_fixtures.h" +#include "../support/archive_lazy_fixtures.h" #include "../support/godot_crypto_fixtures.h" #include "../support/godot_fixtures.h" #include "../support/godot_texture_fixtures.h" +#include "../support/zlib_fixture.h" #include #include @@ -710,6 +712,245 @@ namespace test std::filesystem::remove(invalid_archive_path, error); } + namespace + { + class rpa1_pair final + { + public: + rpa1_pair() + { + const auto fixtures = std::filesystem::path(__FILE__).parent_path().parent_path() / "data/renpy"; + std::filesystem::copy_file(fixtures / "rpa1-split.rpa", data); + std::filesystem::copy_file(fixtures / "rpa1-split.rpi", index); + } + + support::temporary_directory directory{"rpa1"}; + std::filesystem::path data = directory.path() / L"архив.extra.rpa"; + std::filesystem::path index = directory.path() / L"архив.extra.rpi"; + }; + + void write_rpa1_bytes(const std::filesystem::path &path, const std::span bytes) + { + std::ofstream output(path, std::ios::binary); + REQUIRE(output.is_open()); + output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + REQUIRE(output.good()); + } + + void write_rpa1_pickle(const std::filesystem::path &path, const std::string_view pickle) + { + write_rpa1_bytes(path, support::compress_zlib_fixture(std::as_bytes(std::span(pickle)))); + } + } // namespace + + TEST_CASE("module ABI: RPA1 requires the exact adjacent index and data extension", "[contract][rpa1]") + { + const auto scenario = GENERATE(0, 1, 2, 3); + CAPTURE(scenario); + rpa1_pair pair; + if (scenario == 0) { + REQUIRE(std::filesystem::remove(pair.index)); + } else if (scenario == 1) { + std::filesystem::rename(pair.index, pair.directory.path() / "other.rpi"); + } else if (scenario == 2) { + auto renamed = pair.data; + renamed.replace_extension(".bin"); + std::filesystem::rename(pair.data, renamed); + pair.data = renamed; + } else { + pair.data = pair.index; + } + c_module loaded("renpy.so"); + CHECK(loaded.open(pair.data) == false); + } + + TEST_CASE("module ABI: RPA1 rejects unreadable and corrupt sidecars while opening", "[contract][rpa1]") + { + const auto scenario = GENERATE(0, 1, 2, 3, 4, 5, 6); + CAPTURE(scenario); + rpa1_pair pair; + if (scenario == 0) { + write_rpa1_bytes(pair.index, {}); + } else if (scenario == 1) { + std::filesystem::resize_file(pair.index, 5); + } else if (scenario == 2) { + write_rpa1_pickle(pair.index, "N."); + } else if (scenario == 3) { + std::filesystem::resize_file(pair.index, 64ULL * 1024 * 1024 + 1); + } else if (scenario == 4) { + REQUIRE(std::filesystem::remove(pair.index)); + REQUIRE(std::filesystem::create_directory(pair.index)); + } else { + write_rpa1_pickle(pair.index, scenario == 5 ? "Ibad\n." : "I999999999999999999999999999999\n."); + } + c_module loaded("renpy.so"); + CHECK(loaded.open(pair.data) == false); + } + + TEST_CASE("module ABI: RPA1 snapshots its index and accepts uppercase data suffixes", "[contract][rpa1]") + { + rpa1_pair pair; + auto renamed = pair.data; + renamed.replace_extension(".RPA"); + std::filesystem::rename(pair.data, renamed); + c_module loaded("renpy.so"); + REQUIRE(loaded.open(renamed)); + REQUIRE(std::filesystem::remove(pair.index)); + const auto files = loaded.list_files(); + REQUIRE(files.size() == 2); + const auto entry = std::ranges::find_if(files, [](const file &item) { return item.path == L"dir\\hello.txt"; }); + REQUIRE(entry != files.end()); + const temporary_output_file output(pair.directory.path() / "output.txt"); + entry->extract(output.path()); + std::ifstream input(output.path(), std::ios::binary); + const std::string contents{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + CHECK(contents == "first:second:third"); + CHECK(entry->uncompressed_size == 18); + CHECK(entry->compressed_size == 18); + } + + TEST_CASE("module ABI: RPA1 permits an empty archive and zero byte data", "[contract][rpa1]") + { + rpa1_pair pair; + write_rpa1_bytes(pair.data, {}); + write_rpa1_pickle(pair.index, "}."); + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + CHECK(loaded.list_files().empty()); + } + + TEST_CASE("module ABI: RPA1 streams large fragments and reports exact progress", "[contract][rpa1]") + { + rpa1_pair pair; + const std::string contents(128 * 1024 + 7, 'x'); + write_rpa1_bytes(pair.data, std::as_bytes(std::span(contents))); + std::string pickle = "}U"; + pickle.push_back(12); + pickle.append("файл.txt]I0\nI131079\n") + .append("\x86" + "as."); + write_rpa1_pickle(pair.index, pickle); + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + const auto files = loaded.list_files(); + REQUIRE(files.size() == 1); + CHECK(files.front().path == L"файл.txt"); + std::int64_t progress = 0; + const temporary_output_file output(pair.directory.path() / "output.txt"); + const auto callback = [](void *context, const int64_t amount) -> int { + *static_cast(context) += amount; + return TRUE; + }; + CHECK(loaded.extract_file_status(0, output.path(), callback, &progress) == SER_SUCCESS); + CHECK(progress == static_cast(contents.size())); + std::ifstream input(output.path(), std::ios::binary); + const std::string actual{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + CHECK(actual == contents); + } + + TEST_CASE("module ABI: RPA1 extraction cleans partial output on source and callback failures", "[contract][rpa1]") + { + const auto scenario = GENERATE(0, 1, 2); + CAPTURE(scenario); + rpa1_pair pair; + if (scenario == 0) { + const std::string body(std::size_t{256} * 1024, 'x'); + write_rpa1_bytes(pair.data, std::as_bytes(std::span(body))); + std::string pickle = "}U"; + pickle.push_back(13); + pickle.append("dir/hello.txt]I131072\nI6\n") + .append("\x86" + "as."); + write_rpa1_pickle(pair.index, pickle); + } + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + const auto files = loaded.list_files(); + const auto entry = std::ranges::find_if(files, [](const file &item) { return item.path == L"dir\\hello.txt"; }); + REQUIRE(entry != files.end()); + const auto item_index = static_cast(std::distance(files.begin(), entry)); + const temporary_output_file output(pair.directory.path() / "output.txt"); + const auto callback = [](void *context, int64_t) -> int { + if (*static_cast(context) == 2) { + throw std::runtime_error("RPA1 callback failure"); + } + return *static_cast(context) == 0 ? TRUE : FALSE; + }; + if (scenario == 0) { + const locked_source_range locked(pair.data, 128ULL * 1024, 6); + CHECK(loaded.extract_file_status(item_index, output.path(), callback, const_cast(&scenario)) == + SER_ERROR_READ); + } else { + const auto expected = scenario == 1 ? SER_USERABORT : SER_ERROR_SYSTEM; + CHECK(loaded.extract_file_status(item_index, output.path(), callback, const_cast(&scenario)) == + expected); + } + CHECK(std::filesystem::exists(output.path()) == false); + } + + TEST_CASE("module ABI: modern RPA ignores an adjacent RPA1 index", "[contract][rpa1]") + { + const auto sidecar = GENERATE(false, true); + rpa1_pair pair; + const auto contents = support::make_renpy_archive("modern.txt", "modern payload"); + write_rpa1_bytes(pair.data, std::as_bytes(std::span(contents))); + if (sidecar) { + write_rpa1_bytes(pair.index, {}); + } else { + REQUIRE(std::filesystem::remove(pair.index)); + } + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + const auto files = loaded.list_files(); + REQUIRE(files.size() == 1); + CHECK(files.front().path == L"modern.txt"); + } + + TEST_CASE("module ABI: RPA1 payload may begin with a modern signature", "[contract][rpa1][rpa1-collision]") + { + rpa1_pair pair; + const auto contents = + GENERATE(std::string("RPA-payload"), std::string("RPA-2.0 0000000000000000\n"), + std::string("RPA-2.0 0000000000000020\n"), [] { + const auto bytes = support::make_renpy_archive_with_index( + support::byte_buffer{'I', 'b', 'a', 'd', '\n', '.'}); + return std::string(reinterpret_cast(bytes.data()), bytes.size()); + }()); + write_rpa1_bytes(pair.data, std::as_bytes(std::span(contents))); + std::string pickle = "}U"; + pickle.push_back(1); + pickle.append("x]I0\nI") + .append(std::to_string(contents.size())) + .append("\n\x86" + "as."); + write_rpa1_pickle(pair.index, pickle); + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + const auto files = loaded.list_files(); + REQUIRE(files.size() == 1); + const temporary_output_file output(pair.directory.path() / "output.txt"); + files.front().extract(output.path()); + std::ifstream input(output.path(), std::ios::binary); + const std::string actual{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + CHECK(actual == contents); + } + + TEST_CASE("module ABI: modern RPA rejects a truncated supplied signature", "[contract][rpa1]") + { + const support::temporary_archive archive("modern-short-hint", support::make_renpy_archive("x", "body")); + c_module loaded("renpy.so"); + const char prefix = 'R'; + const StorageOpenParams params{.StructSize = sizeof(StorageOpenParams), + .FilePath = archive.path().c_str(), + .Password = nullptr, + .Data = &prefix, + .DataSize = 1}; + HANDLE storage = nullptr; + StorageGeneralInfo info{}; + CHECK(loaded.api().OpenStorage(params, &storage, &info) == SOR_INVALID_FILE); + CHECK(storage == nullptr); + } + TEST_CASE("module ABI: malformed RenPy index shapes are rejected", "[contract]") { const auto expect_rejected = [](const std::string_view label, const support::byte_buffer &pickle_index) { diff --git a/src/tests/integration/archives.cpp b/src/tests/integration/archives.cpp index eef1a22..363fc48 100644 --- a/src/tests/integration/archives.cpp +++ b/src/tests/integration/archives.cpp @@ -9,6 +9,12 @@ #include +TEST_CASE("archives: hermetic RenPy RPA 1.0 sidecar", "[integration][hermetic][rpa1]") +{ + const auto archive = std::filesystem::path(__FILE__).parent_path().parent_path() / "data/renpy/rpa1-split.rpa"; + test::test_archive(archive, {{L"dir\\hello.txt", "first:second:third"}, {L"empty.txt", ""}}); +} + TEST_CASE("archives: hermetic RenPy RPA 2.0", "[integration][hermetic]") { const auto contents = test::support::make_renpy_archive("dir/hello.txt", "renpy payload"); diff --git a/src/tests/unit/pickle.cpp b/src/tests/unit/pickle.cpp index cbc7751..77328d5 100644 --- a/src/tests/unit/pickle.cpp +++ b/src/tests/unit/pickle.cpp @@ -108,8 +108,146 @@ namespace constexpr std::array container_factories{ list_container, tuple_container, dict_container}; + + std::string doubling_memo(const std::size_t depth) + { + std::string result("Nq\0", 3); + for (std::size_t level = 0; level < depth; ++level) { + result.append("h\0", 2); + result.push_back('\x86'); + result.append("q\0", 2); + } + result.push_back('.'); + return result; + } } // namespace +TEST_CASE("pickle: memo amplification is rejected before ownership grows without bound", "[pickle-budget]") +{ + // Each five-byte round doubles the owned subtree without approaching the depth limit. + // The complete legacy parse is finite (about one million live nodes) so this regression + // can demonstrate the missing check without deliberately exhausting the process. + REQUIRE_THROWS_WITH(pickle::loads(doubling_memo(18)), "Pickle node budget exceeded"); +} + +TEST_CASE("pickle: every parsed value is charged to the exact node boundary", "[pickle-budget]") +{ + struct node_case final + { + std::string bytes; + std::size_t nodes; + pickle::value::type type; + }; + const std::vector cases{ + {"N.", 1, pickle::value::type::none}, + {std::string("\x88.", 2), 1, pickle::value::type::bool_}, + {std::string("\x89.", 2), 1, pickle::value::type::bool_}, + {std::string("J\x01\0\0\0.", 6), 1, pickle::value::type::int64}, + {std::string("K\x01.", 3), 1, pickle::value::type::int64}, + {std::string("M\x01\0.", 4), 1, pickle::value::type::int64}, + {"I42\n.", 1, pickle::value::type::int64}, + {std::string("G\x3f\xf0\0\0\0\0\0\0.", 10), 1, pickle::value::type::float64}, + {std::string("U\x01s.", 4), 1, pickle::value::type::string}, + {std::string("T\x01\0\0\0s.", 7), 1, pickle::value::type::string}, + {std::string("\x8c\x01s.", 4), 1, pickle::value::type::string}, + {std::string("X\x01\0\0\0s.", 7), 1, pickle::value::type::string}, + {std::string("C\x01s.", 4), 1, pickle::value::type::bytes}, + {std::string("B\x01\0\0\0s.", 7), 1, pickle::value::type::bytes}, + {"].", 1, pickle::value::type::list}, + {"(Nl.", 2, pickle::value::type::list}, + {").", 1, pickle::value::type::tuple}, + {"(Nt.", 2, pickle::value::type::tuple}, + {std::string("N\x85.", 3), 2, pickle::value::type::tuple}, + {std::string("NN\x86.", 4), 3, pickle::value::type::tuple}, + {std::string("NNN\x87.", 5), 4, pickle::value::type::tuple}, + {"}.", 1, pickle::value::type::dict}, + {std::string("(U\x01kNd.", 7), 3, pickle::value::type::dict}, + {std::string("\x8a\0.", 3), 1, pickle::value::type::int64}, + {std::string("\x8a\x01\x07.", 4), 1, pickle::value::type::int64}, + }; + for (const auto &item : cases) { + CAPTURE(item.bytes, item.nodes); + pickle::limits limits; + limits.max_nodes = item.nodes; + REQUIRE(pickle::loads(item.bytes, limits)->get_type() == item.type); + --limits.max_nodes; + REQUIRE_THROWS_WITH(pickle::loads(item.bytes, limits), "Pickle node budget exceeded"); + } +} + +TEST_CASE("pickle: all memo opcodes share the parsing node budget", "[pickle-budget]") +{ + const std::array stores{ + {std::string("q\0", 2), std::string("r\0\0\0\0", 5), std::string(1, '\x94')}}; + const std::array gets{{std::string("h\0", 2), std::string("j\0\0\0\0", 5)}}; + for (const auto &store : stores) { + for (const auto &get : gets) { + std::string bytes = "N"; + bytes.append(store); + bytes.append(get); + bytes.append("\x86.", 2); + pickle::limits limits; + limits.max_nodes = 4; + REQUIRE(pickle::loads(bytes, limits)->as_tuple().size() == 2); + for (const auto budget : {std::size_t{1}, std::size_t{2}, std::size_t{3}}) { + CAPTURE(budget); + limits.max_nodes = budget; + REQUIRE_THROWS_WITH(pickle::loads(bytes, limits), "Pickle node budget exceeded"); + } + } + } + pickle::limits limits; + limits.max_nodes = 10; + REQUIRE_THROWS_WITH(pickle::loads(doubling_memo(3), limits), "Pickle node budget exceeded"); +} + +TEST_CASE("pickle: owned string and dictionary key copies have an aggregate byte budget", "[pickle-budget]") +{ + const std::array cases{std::string("U\x03" + "foo.", + 6), + std::string("C\x03" + "foo.", + 6), + std::string("I123\n.", 6), + std::string("(U\x01kNd.", 7), + std::string("}U\x01kNs.", 7), + std::string("}(U\x01kNu.", 8), + std::string("U\x01sq\0h\0\x86.", 9)}; + constexpr std::array required_bytes{3, 3, 3, 2, 2, 2, 3}; + for (std::size_t index = 0; index < cases.size(); ++index) { + CAPTURE(index); + pickle::limits limits; + limits.max_owned_bytes = required_bytes[index]; + REQUIRE(pickle::loads(cases[index], limits) != nullptr); + --limits.max_owned_bytes; + REQUIRE_THROWS_WITH(pickle::loads(cases[index], limits), "Pickle owned byte budget exceeded"); + } + const std::string no_bytes = "N."; + pickle::limits zero_bytes; + zero_bytes.max_owned_bytes = 0; + REQUIRE(pickle::loads(no_bytes, zero_bytes)->get_type() == pickle::value::type::none); + const auto span = std::as_bytes(std::span(cases[0])); + REQUIRE_THROWS_WITH(pickle::loads(span, zero_bytes), "Pickle owned byte budget exceeded"); +} + +TEST_CASE("pickle: recursive public clones share exact node and owned byte budgets", "[pickle-budget]") +{ + const auto tree = pickle::loads(std::string("}U\x01k]U\x02hi" + "as.")); + pickle::limits limits; + limits.max_nodes = 3; + limits.max_owned_bytes = 3; + const auto copy = pickle::clone(*tree, limits); + REQUIRE(copy->as_dict().at("k")->as_list()[0]->as_string() == "hi"); + REQUIRE(copy->as_dict().at("k").get() != tree->as_dict().at("k").get()); + limits.max_nodes = 2; + REQUIRE_THROWS_WITH(pickle::clone(*tree, limits), "Pickle node budget exceeded"); + limits.max_nodes = 3; + limits.max_owned_bytes = 2; + REQUIRE_THROWS_WITH(pickle::clone(*tree, limits), "Pickle owned byte budget exceeded"); +} + TEST_CASE("pickle: container nesting is bounded before ownership grows") { for (const auto &encoding : container_encodings) { diff --git a/src/tests/unit/renpy_rpa1_index.cpp b/src/tests/unit/renpy_rpa1_index.cpp new file mode 100644 index 0000000..ec920b5 --- /dev/null +++ b/src/tests/unit/renpy_rpa1_index.cpp @@ -0,0 +1,245 @@ +#include "../../modules/renpy/rpa1_index.h" +#include "../support/zlib_fixture.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace +{ + using unpackrat::renpy::parse_rpa1_index; + using unpackrat::renpy::rpa1_fragment; + using unpackrat::renpy::rpa1_limits; + + struct fixture_entry final + { + std::string path; + std::vector fragments; + }; + + void append_string(std::string &pickle, const std::string_view value) + { + pickle.push_back('X'); + const auto size = static_cast(value.size()); + for (unsigned shift = 0; shift < 32; shift += 8) { + pickle.push_back(static_cast((size >> shift) & 0xffU)); + } + pickle.append(value); + } + + std::string raw_index(const std::vector &entries) + { + std::string pickle = "}"; + for (const auto &entry : entries) { + append_string(pickle, entry.path); + pickle.push_back(']'); + for (const auto &fragment : entry.fragments) { + pickle.append("I").append(std::to_string(fragment.offset)).append("\n"); + pickle.append("I").append(std::to_string(fragment.size)).append("\n"); + pickle.push_back(static_cast(0x86)); + pickle.push_back('a'); + } + pickle.push_back('s'); + } + pickle.push_back('.'); + return pickle; + } + + std::vector compressed_pickle(const std::string_view pickle) + { + return test::support::compress_zlib_fixture(std::as_bytes(std::span(pickle.data(), pickle.size()))); + } + + std::vector index(const std::vector &entries) + { + return compressed_pickle(raw_index(entries)); + } + + std::vector malformed_value(const std::string_view value) + { + std::string pickle = "}"; + append_string(pickle, "file"); + pickle.append(value).append("s."); + return compressed_pickle(pickle); + } +} // namespace + +TEST_CASE("RenPy RPA1 index: fragments retain their order and logical sizes", "[renpy][rpa1]") +{ + const auto result = + parse_rpa1_index(index({{"dir/hello.txt", {{8, 2}, {0, 3}, {10, 0}}}, {"other\\file.bin", {{3, 5}}}}), 10); + REQUIRE(result.size() == 2); + const auto found = std::ranges::find(result, "dir/hello.txt", &unpackrat::renpy::rpa1_entry::path); + REQUIRE(found != result.end()); + CHECK(found->size == 5); + REQUIRE(found->fragments.size() == 3); + CHECK(found->fragments[0].offset == 8); + CHECK(found->fragments[0].size == 2); + CHECK(found->fragments[1].offset == 0); + CHECK(found->fragments[1].size == 3); + CHECK(found->fragments[2].offset == 10); + CHECK(found->fragments[2].size == 0); + const auto other = std::ranges::find(result, "other\\file.bin", &unpackrat::renpy::rpa1_entry::path); + REQUIRE(other != result.end()); + CHECK(other->size == 5); +} + +TEST_CASE("RenPy RPA1 index: empty archives and empty entries are valid", "[renpy][rpa1]") +{ + CHECK(parse_rpa1_index(index({}), 0).empty()); + const auto result = parse_rpa1_index(index({{"empty", {{0, 0}}}}), 0); + REQUIRE(result.size() == 1); + CHECK(result[0].path == "empty"); + CHECK(result[0].size == 0); + REQUIRE(result[0].fragments.size() == 1); + CHECK(result[0].fragments[0].offset == 0); + CHECK(result[0].fragments[0].size == 0); + CHECK_THROWS_AS(parse_rpa1_index(index({}), -1), std::runtime_error); +} + +TEST_CASE("RenPy RPA1 index: compressed and expanded budgets have exact boundaries", "[renpy][rpa1]") +{ + const std::vector entries{{"file", {{0, 1}}}}; + const auto pickle = raw_index(entries); + const auto compressed = compressed_pickle(pickle); + rpa1_limits limits; + limits.max_compressed_index_bytes = compressed.size(); + limits.max_decompressed_index_bytes = pickle.size(); + const auto accepted = parse_rpa1_index(compressed, 1, limits); + REQUIRE(accepted.size() == 1); + CHECK(accepted[0].size == 1); + + --limits.max_compressed_index_bytes; + CHECK_THROWS_AS(parse_rpa1_index(compressed, 1, limits), std::runtime_error); + ++limits.max_compressed_index_bytes; + --limits.max_decompressed_index_bytes; + CHECK_THROWS_AS(parse_rpa1_index(compressed, 1, limits), std::runtime_error); +} + +TEST_CASE("RenPy RPA1 index: corrupt compressed data and malformed pickle fail closed", "[renpy][rpa1]") +{ + CHECK_THROWS_AS(parse_rpa1_index({}, 1), std::runtime_error); + const std::array invalid{std::byte{'n'}, std::byte{'o'}}; + CHECK_THROWS_AS(parse_rpa1_index(invalid, 1), std::runtime_error); + auto truncated = index({{"file", {{0, 1}}}}); + truncated.pop_back(); + CHECK_THROWS_AS(parse_rpa1_index(truncated, 1), std::runtime_error); + auto trailing = index({{"file", {{0, 1}}}}); + trailing.push_back(std::byte{0}); + CHECK_THROWS_AS(parse_rpa1_index(trailing, 1), std::runtime_error); + CHECK_THROWS_AS(parse_rpa1_index(compressed_pickle("N."), 1), std::runtime_error); + CHECK_THROWS_AS(parse_rpa1_index(compressed_pickle("?"), 1), std::runtime_error); + CHECK_THROWS_AS(parse_rpa1_index(compressed_pickle("Ibad\n."), 1), std::runtime_error); + CHECK_THROWS_AS(parse_rpa1_index(compressed_pickle("I999999999999999999999999999999\n."), 1), std::runtime_error); +} + +TEST_CASE("RenPy RPA1 index: property lists require nonempty integer pairs", "[renpy][rpa1]") +{ + const std::array invalid_values{ + "N", "]", "]Na", "])a", "](I0\nta", "](I0\nI1\nI2\nta", "](NI1\nta", "](I0\nNta"}; + for (const auto value : invalid_values) { + CAPTURE(value); + CHECK_THROWS_AS(parse_rpa1_index(malformed_value(value), 10), std::runtime_error); + } +} + +TEST_CASE("RenPy RPA1 index: entry and aggregate fragment limits are enforced", "[renpy][rpa1]") +{ + const auto two_entries = index({{"first", {{0, 1}}}, {"second", {{1, 1}}}}); + rpa1_limits limits; + limits.max_entries = 2; + limits.max_fragments = 2; + CHECK(parse_rpa1_index(two_entries, 2, limits).size() == 2); + limits.max_entries = 1; + CHECK_THROWS_AS(parse_rpa1_index(two_entries, 2, limits), std::runtime_error); + limits.max_entries = 2; + limits.max_fragments = 1; + CHECK_THROWS_AS(parse_rpa1_index(two_entries, 2, limits), std::runtime_error); + limits.max_fragments = 2; + CHECK_THROWS_AS(parse_rpa1_index(index({{"one", {{0, 1}, {1, 1}, {2, 1}}}}), 3, limits), std::runtime_error); + limits.max_entries = 0; + limits.max_fragments = 0; + CHECK(parse_rpa1_index(index({}), 0, limits).empty()); +} + +TEST_CASE("RenPy RPA1 index: range decision table rejects invalid extents", "[renpy][rpa1]") +{ + struct range_case final + { + std::int64_t offset; + std::int64_t size; + bool accepted; + }; + constexpr std::array cases{ + range_case{0, 0, true}, + range_case{0, 10, true}, + range_case{10, 0, true}, + range_case{9, 1, true}, + range_case{-1, 0, false}, + range_case{0, -1, false}, + range_case{11, 0, false}, + range_case{0, 11, false}, + range_case{9, 2, false}, + range_case{std::numeric_limits::max(), 1, false}, + range_case{1, std::numeric_limits::max(), false}, + }; + for (const auto &item : cases) { + CAPTURE(item.offset, item.size); + const auto compressed = index({{"file", {{item.offset, item.size}}}}); + if (item.accepted) { + const auto result = parse_rpa1_index(compressed, 10); + REQUIRE(result.size() == 1); + CHECK(result[0].size == item.size); + } else { + CHECK_THROWS_AS(parse_rpa1_index(compressed, 10), std::runtime_error); + } + } +} + +TEST_CASE("RenPy RPA1 index: fragment sums are checked before signed overflow", "[renpy][rpa1]") +{ + constexpr auto maximum = std::numeric_limits::max(); + const auto accepted = parse_rpa1_index(index({{"file", {{0, maximum - 1}, {0, 1}}}}), maximum); + REQUIRE(accepted.size() == 1); + CHECK(accepted[0].size == maximum); + CHECK_THROWS_AS(parse_rpa1_index(index({{"file", {{0, maximum}, {0, 1}}}}), maximum), std::runtime_error); +} + +TEST_CASE("RenPy RPA1 index: paths must remain relative and free of traversal", "[renpy][rpa1]") +{ + const std::vector rejected{ + "", "/root", "\\root", "C:file", "dir/file:stream", "..", + "../file", "dir/../file", "dir\\..\\file", "dir/..", "dir\\..", std::string("dir\0file", 8)}; + for (const auto &path : rejected) { + CAPTURE(path); + CHECK_THROWS_AS(parse_rpa1_index(index({{path, {{0, 0}}}}), 0), std::runtime_error); + } + const std::array accepted{"file", "dir/file", "dir\\file", "...", "a..b", "dir/./file"}; + for (const auto path : accepted) { + CAPTURE(path); + const auto result = parse_rpa1_index(index({{std::string(path), {{0, 0}}}}), 0); + REQUIRE(result.size() == 1); + CHECK(result[0].path == path); + } +} + +TEST_CASE("RenPy RPA1 index: path byte budgets preserve valid UTF8", "[renpy][rpa1]") +{ + const std::string unicode = "dir/\xd0\xbf\xd1\x83\xd1\x82\xd1\x8c.txt"; + rpa1_limits limits; + limits.max_path_bytes = unicode.size(); + const auto result = parse_rpa1_index(index({{unicode, {{0, 0}}}}), 0, limits); + REQUIRE(result.size() == 1); + CHECK(result[0].path == unicode); + --limits.max_path_bytes; + CHECK_THROWS_AS(parse_rpa1_index(index({{unicode, {{0, 0}}}}), 0, limits), std::runtime_error); +}