From 87a8e00812d59d5b3ab7bb707f0b578c201439e7 Mon Sep 17 00:00:00 2001 From: Roman Kharitonov Date: Wed, 21 Oct 2026 23:14:24 +1000 Subject: [PATCH] Support RenPy RPA 1.0 sidecar archives Open headerless .rpa data with its adjacent .rpi index and stream ordered fragments with checked paths, extents, and metadata budgets. Bound Pickle memo copying and cover the parser and real Observer ABI with authored fixtures. Refs #77 Co-Authored-By: Codex (gpt-6.1-sol) --- README.md | 2 +- build/projects/fuzz-renpy.vcxproj | 2 + build/projects/renpy.vcxproj | 2 + build/projects/tests.vcxproj | 2 + .../tests/test_supported_architecture_docs.py | 67 ++++- src/modules/renpy/ChangeLog.en.txt | 6 + src/modules/renpy/ChangeLog.ru.txt | 6 + src/modules/renpy/VERSION | 2 +- src/modules/renpy/pickle.cpp | 135 +++++++--- src/modules/renpy/pickle.h | 18 +- src/modules/renpy/renpy.cpp | 133 +++++++++- src/modules/renpy/rpa1_index.cpp | 110 ++++++++ src/modules/renpy/rpa1_index.h | 43 +++ src/tests/data/renpy/README.md | 37 ++- src/tests/data/renpy/rpa1-split.rpa | Bin 0 -> 29 bytes src/tests/data/renpy/rpa1-split.rpi | Bin 0 -> 85 bytes src/tests/framework/observer.cpp | 241 +++++++++++++++++ src/tests/integration/archives.cpp | 6 + src/tests/unit/pickle.cpp | 138 ++++++++++ src/tests/unit/renpy_rpa1_index.cpp | 245 ++++++++++++++++++ 20 files changed, 1136 insertions(+), 59 deletions(-) create mode 100644 src/modules/renpy/rpa1_index.cpp create mode 100644 src/modules/renpy/rpa1_index.h create mode 100644 src/tests/data/renpy/rpa1-split.rpa create mode 100644 src/tests/data/renpy/rpa1-split.rpi create mode 100644 src/tests/unit/renpy_rpa1_index.cpp diff --git a/README.md b/README.md index aa40546..aebce25 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ from the module's release listing below. | Module | Formats | Release | ChangeLog | |--------|---------|---------|-----------| -| Ren'Py | `.rpa` (RPA-2.0, RPA-3.0) | [renpy](https://github.com/refaim/unpackrat/releases?q=renpy) | [RU](src/modules/renpy/ChangeLog.ru.txt) [EN](src/modules/renpy/ChangeLog.en.txt) | +| Ren'Py | `.rpa` (RPA-1.0 with adjacent `.rpi`, RPA-2.0, RPA-3.0) | [renpy](https://github.com/refaim/unpackrat/releases?q=renpy) | [RU](src/modules/renpy/ChangeLog.ru.txt) [EN](src/modules/renpy/ChangeLog.en.txt) | | RPG Maker | `.rgssad` (XP, RGSS1), `.rgss2a` (VX, RGSS2), `.rgss3a` (VX Ace, RGSS3) | [rpgmaker](https://github.com/refaim/unpackrat/releases?q=rpgmaker) | [RU](src/modules/rpgmaker/ChangeLog.ru.txt) [EN](src/modules/rpgmaker/ChangeLog.en.txt) | | Zanzarah | `.pak` | [zanzarah 2.3.0](https://github.com/refaim/unpackrat/releases?q=zanzarah) | [RU](src/modules/zanzarah/ChangeLog.ru.txt) [EN](src/modules/zanzarah/ChangeLog.en.txt) | | Godot | `.pck` | [godot 1.2.0](https://github.com/refaim/unpackrat/releases?q=godot) | [RU](src/modules/godot/ChangeLog.ru.txt) [EN](src/modules/godot/ChangeLog.en.txt) | diff --git a/build/projects/fuzz-renpy.vcxproj b/build/projects/fuzz-renpy.vcxproj index ebb7ac4..383e142 100644 --- a/build/projects/fuzz-renpy.vcxproj +++ b/build/projects/fuzz-renpy.vcxproj @@ -30,12 +30,14 @@ + + diff --git a/build/projects/renpy.vcxproj b/build/projects/renpy.vcxproj index 25c5608..d965e28 100644 --- a/build/projects/renpy.vcxproj +++ b/build/projects/renpy.vcxproj @@ -34,6 +34,7 @@ + @@ -42,6 +43,7 @@ + diff --git a/build/projects/tests.vcxproj b/build/projects/tests.vcxproj index c058368..8323b9f 100644 --- a/build/projects/tests.vcxproj +++ b/build/projects/tests.vcxproj @@ -102,6 +102,8 @@ $(IntDir)pickle_unit.obj + + $(IntDir)bounded_stream_unit.obj diff --git a/build/tests/test_supported_architecture_docs.py b/build/tests/test_supported_architecture_docs.py index c43aec5..0d70289 100644 --- a/build/tests/test_supported_architecture_docs.py +++ b/build/tests/test_supported_architecture_docs.py @@ -26,6 +26,12 @@ DOCUMENTS = (README, AGENTS, BUILD_SYSTEM, METHODOLOGY) MODULES = ("renpy", "rpgmaker", "zanzarah", "godot") +CONSOLIDATED_RELEASES = { + "renpy": "3.2.0", + "rpgmaker": "1.3.0", + "zanzarah": "2.3.0", + "godot": "1.2.0", +} LANGUAGES = ("en", "ru") @@ -177,6 +183,52 @@ def test_methodology_describes_the_real_binary_gate_without_removed_checks(self) self.assertIn("forbidden imports", docs) self.assertIn("BinSkim", docs) + def test_future_changelog_release_keeps_the_historical_windows_announcement(self) -> None: + for language, header, announcement in ( + ("en", "Version", "+ Added support for Windows XP–11."), + ("ru", "Версия", "+ Добавлена поддержка Windows XP–11."), + ): + with self.subTest(language=language): + changelog = ( + f"{header} 99.0.0\n-------------\n + Added a future feature.\n\n" + f"{header} 3.2.0\n-------------\n {announcement}\n\n" + f"{header} 1.0.0\n-------------\n + Original release.\n" + ) + case = SupportedArchitectureDocumentTests( + "test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history" + ) + with ( + mock.patch(__name__ + ".MODULES", ("renpy",)), + mock.patch(__name__ + ".LANGUAGES", (language,)), + mock.patch(__name__ + ".read", return_value=changelog), + mock.patch.object(Path, "read_text", return_value="99.0.0"), + ): + case.test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history() + + def test_latest_or_other_release_cannot_replace_the_consolidated_windows_announcement(self) -> None: + for language, header, announcement in ( + ("en", "Version", "+ Added support for Windows XP–11."), + ("ru", "Версия", "+ Добавлена поддержка Windows XP–11."), + ): + with self.subTest(language=language): + changelog = ( + f"{header} 99.0.0\n-------------\n {announcement}\n\n" + f"{header} 3.2.0\n-------------\n + A different historical feature.\n\n" + f"{header} 3.1.0\n-------------\n {announcement}\n\n" + f"{header} 1.0.0\n-------------\n + Original release.\n" + ) + case = SupportedArchitectureDocumentTests( + "test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history" + ) + with ( + mock.patch(__name__ + ".MODULES", ("renpy",)), + mock.patch(__name__ + ".LANGUAGES", (language,)), + mock.patch(__name__ + ".read", return_value=changelog), + mock.patch.object(Path, "read_text", return_value="99.0.0"), + self.assertRaisesRegex(AssertionError, "XP"), + ): + case.test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history() + def test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history(self) -> None: headers = {"en": "Version", "ru": "\u0412\u0435\u0440\u0441\u0438\u044f"} for module in MODULES: @@ -203,8 +255,19 @@ def test_changelogs_drop_the_arm_build_claim_and_keep_the_version_history(self) line for line in body[:end] if line.lstrip().startswith(("*", "+")) ] self.assertGreaterEqual(len(entries), 1) - # Every consolidated current release announces Windows XP..11 support; - # a historical announcement alone does not satisfy the release contract. + # The Windows support announcement belongs to its original consolidated + # release. New releases keep that exact historical section without + # repeating the announcement as if Windows support were newly added. + consolidated = f"{headers[language]} {CONSOLIDATED_RELEASES[module]}" + self.assertIn(consolidated, lines) + body = lines[lines.index(consolidated) + 2:] + end = next( + (index for index, line in enumerate(body) if line.startswith(headings)), + len(body), + ) + entries = [ + line for line in body[:end] if line.lstrip().startswith(("*", "+")) + ] announcement = { "en": "+ Added support for Windows XP–11.", "ru": "+ Добавлена поддержка Windows XP–11.", diff --git a/src/modules/renpy/ChangeLog.en.txt b/src/modules/renpy/ChangeLog.en.txt index 648dfe4..f8aa4f8 100644 --- a/src/modules/renpy/ChangeLog.en.txt +++ b/src/modules/renpy/ChangeLog.en.txt @@ -1,3 +1,9 @@ +Version 3.3.0 +------------- + + Added support for RPA 1.0 archives with adjacent .rpi indexes, including files stored in multiple fragments. + * Validate legacy index paths, fragment bounds and metadata limits before opening an archive. + * Limit object and string copying while parsing Pickle indexes to reject memo amplification safely. + Version 3.2.0 ------------- + Added support for Windows XP–11. diff --git a/src/modules/renpy/ChangeLog.ru.txt b/src/modules/renpy/ChangeLog.ru.txt index d4883c4..eaecfcc 100644 --- a/src/modules/renpy/ChangeLog.ru.txt +++ b/src/modules/renpy/ChangeLog.ru.txt @@ -1,3 +1,9 @@ +Версия 3.3.0 +------------- + + Добавлена поддержка архивов RPA 1.0 с соседним индексом .rpi, включая файлы из нескольких фрагментов. + * До открытия архива проверяются пути, границы фрагментов и ограничения размера старого индекса. + * Ограничено копирование объектов и строк в индексах Pickle для защиты от раздувания через memo. + Версия 3.2.0 ------------- + Добавлена поддержка Windows XP–11. diff --git a/src/modules/renpy/VERSION b/src/modules/renpy/VERSION index 944880f..15a2799 100644 --- a/src/modules/renpy/VERSION +++ b/src/modules/renpy/VERSION @@ -1 +1 @@ -3.2.0 +3.3.0 diff --git a/src/modules/renpy/pickle.cpp b/src/modules/renpy/pickle.cpp index e471fc8..39d8a8f 100644 --- a/src/modules/renpy/pickle.cpp +++ b/src/modules/renpy/pickle.cpp @@ -64,49 +64,78 @@ namespace pickle namespace { - list clone_list(const list &source) + void consume_node(limits &remaining) + { + if (remaining.max_nodes == 0) { + throw std::runtime_error("Pickle node budget exceeded"); + } + --remaining.max_nodes; + } + + void consume_bytes(limits &remaining, const std::size_t size) + { + if (size > remaining.max_owned_bytes) { + throw std::runtime_error("Pickle owned byte budget exceeded"); + } + remaining.max_owned_bytes -= size; + } + + value_ptr clone_value(const value &source, limits &remaining); + + list clone_list(const list &source, limits &remaining) { list result; - result.reserve(source.size()); - std::ranges::transform(source, std::back_inserter(result), [](const auto &item) { return clone(*item); }); + result.reserve(std::min(source.size(), remaining.max_nodes)); + std::ranges::transform(source, std::back_inserter(result), + [&remaining](const auto &item) { return clone_value(*item, remaining); }); return result; } - dict clone_dict(const dict &source) + dict clone_dict(const dict &source, limits &remaining) { dict result; - result.reserve(source.size()); + result.reserve(std::min(source.size(), remaining.max_nodes)); for (const auto &[key, item] : source) { - result.emplace(key, clone(*item)); + consume_bytes(remaining, key.size()); + result.emplace(key, clone_value(*item, remaining)); } return result; } + + value_ptr clone_value(const value &source, limits &remaining) + { + consume_node(remaining); + switch (source.get_type()) { + case value::type::none: + return value::none(); + case value::type::bool_: + return value::boolean(source.as_bool()); + case value::type::int64: + return value::int64(source.as_int64()); + case value::type::float64: + return value::float64(source.as_float64()); + case value::type::bytes: + consume_bytes(remaining, source.as_string().size()); + return value::bytes(source.as_string()); + case value::type::string: + consume_bytes(remaining, source.as_string().size()); + return value::string(source.as_string()); + case value::type::list: + return value::list(clone_list(source.as_list(), remaining)); + case value::type::dict: + return value::dict(clone_dict(source.as_dict(), remaining)); + case value::type::tuple: + return value::tuple(clone_list(source.as_tuple(), remaining)); + default: + throw std::runtime_error("Unsupported pickle value type"); + } + } } // namespace - value_ptr clone(const value &source) + value_ptr clone(const value &source, const limits &bounds) { - switch (source.get_type()) { - case value::type::none: - return value::none(); - case value::type::bool_: - return value::boolean(source.as_bool()); - case value::type::int64: - return value::int64(source.as_int64()); - case value::type::float64: - return value::float64(source.as_float64()); - case value::type::bytes: - return value::bytes(source.as_string()); - case value::type::string: - return value::string(source.as_string()); - case value::type::list: - return value::list(clone_list(source.as_list())); - case value::type::dict: - return value::dict(clone_dict(source.as_dict())); - case value::type::tuple: - return value::tuple(clone_list(source.as_tuple())); - default: - throw std::runtime_error("Unsupported pickle value type"); - } + auto remaining = bounds; + return clone_value(source, remaining); } // Pickle opcodes (subset needed for basic functionality) @@ -206,6 +235,7 @@ namespace pickle if (pos_ + length > data_.size()) { throw std::runtime_error("Unexpected end of pickle data"); } + consume_bytes(remaining_, length); std::string result(reinterpret_cast(data_.data() + pos_), length); pos_ += length; return result; @@ -220,6 +250,7 @@ namespace pickle if (pos_ >= data_.size()) { throw std::runtime_error("Unexpected end of pickle data"); } + consume_bytes(remaining_, pos_ - start); std::string result(reinterpret_cast(data_.data() + start), pos_ - start); pos_++; // skip newline return result; @@ -264,26 +295,32 @@ namespace pickle return std::move(stack_[0]); case opcodes::NONE: + consume_node(remaining_); stack_.push_back(value::none()); break; case opcodes::NEWTRUE: + consume_node(remaining_); stack_.push_back(value::boolean(true)); break; case opcodes::NEWFALSE: + consume_node(remaining_); stack_.push_back(value::boolean(false)); break; case opcodes::BININT: + consume_node(remaining_); stack_.push_back(value::int64(static_cast(read_uint32_le()))); break; case opcodes::BININT1: + consume_node(remaining_); stack_.push_back(value::int64(read_byte())); break; case opcodes::BININT2: + consume_node(remaining_); stack_.push_back(value::int64(read_uint16_le())); break; @@ -296,6 +333,7 @@ namespace pickle int_str.pop_back(); // Remove trailing L } int64_t val = std::stoll(int_str); + consume_node(remaining_); stack_.push_back(value::int64(val)); break; } @@ -304,47 +342,55 @@ namespace pickle const uint64_t bits = std::byteswap(read_uint64_le()); double val; std::memcpy(&val, &bits, sizeof(double)); + consume_node(remaining_); stack_.push_back(value::float64(val)); break; } case opcodes::SHORT_BINSTRING: { uint8_t length = read_byte(); + consume_node(remaining_); stack_.push_back(value::string(read_string(length))); break; } case opcodes::BINSTRING: { uint32_t length = read_uint32_le(); + consume_node(remaining_); stack_.push_back(value::string(read_string(length))); break; } case opcodes::SHORT_BINUNICODE: { uint8_t length = read_byte(); + consume_node(remaining_); stack_.push_back(value::string(read_string(length))); break; } case opcodes::BINUNICODE: { uint32_t length = read_uint32_le(); + consume_node(remaining_); stack_.push_back(value::string(read_string(length))); break; } case opcodes::SHORT_BINBYTES: { uint8_t length = read_byte(); + consume_node(remaining_); stack_.push_back(value::bytes(read_string(length))); break; } case opcodes::BINBYTES: { uint32_t length = read_uint32_le(); + consume_node(remaining_); stack_.push_back(value::bytes(read_string(length))); break; } case opcodes::EMPTY_LIST: + consume_node(remaining_); stack_.push_back(value::list({})); break; @@ -377,16 +423,19 @@ namespace pickle case opcodes::LIST: { auto items = pop_to_mark(); + consume_node(remaining_); stack_.push_back(value::list(std::move(items))); break; } case opcodes::EMPTY_TUPLE: + consume_node(remaining_); stack_.push_back(value::tuple({})); break; case opcodes::TUPLE: { auto items = pop_to_mark(); + consume_node(remaining_); stack_.push_back(value::tuple(std::move(items))); break; } @@ -399,6 +448,7 @@ namespace pickle stack_.pop_back(); list tuple_items; tuple_items.push_back(std::move(item)); + consume_node(remaining_); stack_.push_back(value::tuple(std::move(tuple_items))); break; } @@ -414,6 +464,7 @@ namespace pickle list tuple_items; tuple_items.push_back(std::move(item1)); tuple_items.push_back(std::move(item2)); + consume_node(remaining_); stack_.push_back(value::tuple(std::move(tuple_items))); break; } @@ -432,11 +483,13 @@ namespace pickle tuple_items.push_back(std::move(item1)); tuple_items.push_back(std::move(item2)); tuple_items.push_back(std::move(item3)); + consume_node(remaining_); stack_.push_back(value::tuple(std::move(tuple_items))); break; } case opcodes::EMPTY_DICT: + consume_node(remaining_); stack_.push_back(value::dict({})); break; @@ -450,9 +503,11 @@ namespace pickle if (items[i]->get_type() != value::type::string) { throw std::runtime_error("Dict key must be string"); } + consume_bytes(remaining_, items[i]->as_string().size()); std::string key = items[i]->as_string(); dict_data[std::move(key)] = std::move(items[i + 1]); } + consume_node(remaining_); stack_.push_back(value::dict(std::move(dict_data))); break; } @@ -474,6 +529,7 @@ namespace pickle throw std::runtime_error("Dict key must be string"); } + consume_bytes(remaining_, key->as_string().size()); dict_val->set(key->as_string(), std::move(val)); break; } @@ -498,6 +554,7 @@ namespace pickle value::checked_child_depth(items[i + 1]); } for (size_t i = 0; i < items.size(); i += 2) { + consume_bytes(remaining_, items[i]->as_string().size()); std::string key = items[i]->as_string(); dict_val->set(std::move(key), std::move(items[i + 1])); } @@ -509,7 +566,7 @@ namespace pickle if (stack_.empty()) { throw std::runtime_error("No item on stack for BINPUT"); } - memo_[memo_id] = clone(*stack_.back()); + memo_[memo_id] = clone_value(*stack_.back(), remaining_); break; } @@ -518,7 +575,7 @@ namespace pickle if (stack_.empty()) { throw std::runtime_error("No item on stack for LONG_BINPUT"); } - memo_[memo_id] = clone(*stack_.back()); + memo_[memo_id] = clone_value(*stack_.back(), remaining_); break; } @@ -528,7 +585,7 @@ namespace pickle if (it == memo_.end()) { throw std::runtime_error("Memo key not found"); } - stack_.push_back(clone(*it->second)); + stack_.push_back(clone_value(*it->second, remaining_)); break; } @@ -538,7 +595,7 @@ namespace pickle if (it == memo_.end()) { throw std::runtime_error("Memo key not found"); } - stack_.push_back(clone(*it->second)); + stack_.push_back(clone_value(*it->second, remaining_)); break; } @@ -557,6 +614,7 @@ namespace pickle case opcodes::LONG1: { uint8_t length = read_byte(); if (length == 0) { + consume_node(remaining_); stack_.push_back(value::int64(0)); } else { if (length > sizeof(std::int64_t)) { @@ -574,6 +632,7 @@ namespace pickle bits |= ~std::uint64_t{0} << length * 8U; } + consume_node(remaining_); stack_.push_back(value::int64(std::bit_cast(bits))); } break; @@ -584,7 +643,7 @@ namespace pickle throw std::runtime_error("No item on stack for MEMOIZE"); } const auto memo_id = static_cast(memo_.size()); - memo_[memo_id] = clone(*stack_.back()); + memo_[memo_id] = clone_value(*stack_.back(), remaining_); break; } @@ -605,15 +664,15 @@ namespace pickle throw std::runtime_error("Unexpected end of pickle data"); } - value_ptr loads(const std::span data) + value_ptr loads(const std::span data, const limits &bounds) { - parser p(data); + parser p(data, bounds); return p.parse(); } - value_ptr loads(const std::string &data) + value_ptr loads(const std::string &data, const limits &bounds) { const auto byte_span = std::span(reinterpret_cast(data.data()), data.size()); - return loads(byte_span); + return loads(byte_span, bounds); } } // namespace pickle diff --git a/src/modules/renpy/pickle.h b/src/modules/renpy/pickle.h index b95ac89..dd5d7d3 100644 --- a/src/modules/renpy/pickle.h +++ b/src/modules/renpy/pickle.h @@ -19,6 +19,12 @@ namespace pickle using list = std::vector; using dict = std::unordered_map; + struct limits final + { + std::size_t max_nodes = 1'000'000; + std::size_t max_owned_bytes = std::size_t{64} * 1024 * 1024; + }; + class value { public: @@ -197,7 +203,7 @@ namespace pickle } }; - value_ptr clone(const value &source); + value_ptr clone(const value &source, const limits &bounds = {}); class parser { @@ -207,6 +213,9 @@ namespace pickle std::vector stack_; std::vector mark_stack_; std::unordered_map memo_; + // Monotonic work/ownership allowances cover original values, every memo clone, + // and copied strings and dictionary keys, even after a memo slot is replaced. + limits remaining_; uint8_t read_byte(); @@ -227,15 +236,16 @@ namespace pickle value_ptr parse_value(); public: - explicit parser(const std::span data) : data_(data) + explicit parser(const std::span data, const limits &bounds = {}) + : data_(data), remaining_(bounds) { } value_ptr parse(); }; - value_ptr loads(std::span data); + value_ptr loads(std::span data, const limits &bounds = {}); - value_ptr loads(const std::string &data); + value_ptr loads(const std::string &data, const limits &bounds = {}); } // namespace pickle #endif diff --git a/src/modules/renpy/renpy.cpp b/src/modules/renpy/renpy.cpp index 3a49e0e..acb76fe 100644 --- a/src/modules/renpy/renpy.cpp +++ b/src/modules/renpy/renpy.cpp @@ -2,7 +2,12 @@ #include "../../core/io/bounded_stream.h" #include "../extractor.h" #include "pickle.h" +#include "rpa1_index.h" +#include +#include +#include +#include #include #include #include @@ -12,6 +17,56 @@ namespace extractor { + namespace + { + constexpr std::string_view rpa_signature = "RPA-"; + + struct rpa1_storage final : storage_state + { + std::vector entries; + }; + + bool valid_modern_index(extractor &parser, std::istream &stream) + { + try { + static_cast(parser.list_files(stream)); + return true; + } catch (const std::runtime_error &) { + return false; + } catch (const std::out_of_range &) { + return false; + } catch (const std::invalid_argument &) { + return false; + } + } + + void extract_fragments(std::istream &source, std::ostream &output, + const std::vector &fragments, + const std::function &report_progress) + { + constexpr std::int64_t chunk_bytes = 128LL * 1024; + std::vector buffer(static_cast(chunk_bytes)); + for (const auto &fragment : fragments) { + auto position = fragment.offset; + auto remaining = fragment.size; + while (remaining > 0) { + const auto count = std::min(remaining, chunk_bytes); + try { + source.clear(); + source.seekg(position); + source.read(buffer.data(), static_cast(count)); + } catch (const std::ios_base::failure &) { + throw read_error(); + } + output.write(buffer.data(), static_cast(count)); + report_progress(count); + position += count; + remaining -= count; + } + } + } + } // namespace + void configure([[maybe_unused]] const wchar_t *settings) noexcept { } @@ -30,18 +85,55 @@ namespace extractor std::vector extractor::get_signature() { - const std::string str = "RPA-"; - std::vector signature(str.size()); - std::memcpy(signature.data(), str.data(), str.size()); - return signature; + // RPA 1.0 data has no signature. Recognition below requires its exact adjacent index. + return {}; } - archive_info extractor::get_archive_info( - [[maybe_unused]] std::istream &stream, const std::span &data, - [[maybe_unused]] const std::filesystem::path &path, - [[maybe_unused]] const std::string_view password) // NOLINT(*-convert-member-functions-to-static) + archive_info extractor::get_archive_info(std::istream &stream, const std::span &data, + const std::filesystem::path &path, + [[maybe_unused]] const std::string_view password) { - static_cast(data); + unpackrat::io::bounded_stream input(stream); + std::array prefix{}; + const auto prefix_size = + static_cast(std::min(input.size(), static_cast(rpa_signature.size()))); + input.seek_absolute(0); + input.read_exact(prefix.data(), prefix_size); + const bool has_modern_signature = std::string_view(prefix.data(), prefix_size) == rpa_signature; + auto extension = path.extension().wstring(); + std::ranges::transform(extension, extension.begin(), [](const wchar_t value) { return std::towlower(value); }); + auto index_path = path; + index_path.replace_extension(L".rpi"); + + // A headerless RPA1 body may contain any bytes, including a modern header. If both + // representations are present, a valid modern archive wins; otherwise try its sidecar. + if (has_modern_signature && + (extension != L".rpa" || !std::filesystem::exists(index_path) || valid_modern_index(*this, stream))) { + if (!data.empty() && (data.size() < rpa_signature.size() || + std::memcmp(data.data(), rpa_signature.data(), rpa_signature.size()) != 0)) { + throw std::runtime_error("Signature mismatch"); + } + return archive_info{L"RenPy", L"", L""}; + } + + if (extension != L".rpa") { + throw std::runtime_error("RPA 1.0 requires a .rpa data file"); + } + std::ifstream index(index_path, std::ios::binary); + if (!index.is_open()) { + throw std::runtime_error("RPA 1.0 index is missing or unreadable"); + } + unpackrat::io::bounded_stream index_input(index); + const auto index_size = static_cast(index_input.size()); + if (index_size > unpackrat::renpy::rpa1_limits{}.max_compressed_index_bytes) { + throw std::runtime_error("RPA compressed index exceeds the metadata budget"); + } + std::vector compressed_index(static_cast(index_size)); + index_input.seek_absolute(0); + index_input.read_exact(reinterpret_cast(compressed_index.data()), compressed_index.size()); + auto state = std::make_unique(); + state->entries = unpackrat::renpy::parse_rpa1_index(compressed_index, input.size()); + state_ = std::move(state); return archive_info{L"RenPy", L"", L""}; } @@ -64,7 +156,7 @@ namespace extractor unpackrat::io::bounded_stream &stream) { std::string version_check(3, '\0'); - stream.seek_absolute(static_cast(extractor::get_signature().size())); + stream.seek_absolute(static_cast(rpa_signature.size())); stream.read_exact(version_check.data(), version_check.size()); if (version_check == "2.0") { @@ -87,9 +179,26 @@ namespace extractor throw std::runtime_error("Unsupported RPA version"); } - std::vector> extractor::list_files( - std::istream &stream) // NOLINT(*-convert-member-functions-to-static) + std::vector> extractor::list_files(std::istream &stream) { + if (state_) { + const auto &state = static_cast(*state_); + std::vector> files; + files.reserve(state.entries.size()); + for (const auto &entry : state.entries) { + auto item = std::make_unique(); + item->path = entry.path; + item->compressed_body_size_in_bytes = entry.size; + item->uncompressed_body_size_in_bytes = entry.size; + item->extract = [fragments = entry.fragments](std::istream &source, std::ostream &output, + const std::function &report_progress) { + extract_fragments(source, output, fragments, report_progress); + }; + files.push_back(std::move(item)); + } + return files; + } + unpackrat::io::bounded_stream input(stream); const auto [index_offset, decoder] = parse_header(input); const auto archive_size = input.size(); diff --git a/src/modules/renpy/rpa1_index.cpp b/src/modules/renpy/rpa1_index.cpp new file mode 100644 index 0000000..de63ff3 --- /dev/null +++ b/src/modules/renpy/rpa1_index.cpp @@ -0,0 +1,110 @@ +#include "rpa1_index.h" +#include "../../core/compression/zlib_codec.h" +#include "pickle.h" + +#include +#include +#include +#include + +namespace unpackrat::renpy +{ + namespace + { + pickle::value_ptr load_index(const std::span data) + { + try { + return pickle::loads(data); + } catch (const std::invalid_argument &) { + throw std::runtime_error("RPA1 index contains an invalid integer"); + } catch (const std::out_of_range &) { + throw std::runtime_error("RPA1 index contains an integer outside int64"); + } + } + + void validate_path(const std::string_view path, const std::size_t max_path_bytes) + { + if (path.empty()) { + throw std::runtime_error("RPA1 entry path is empty"); + } + if (path.size() > max_path_bytes) { + throw std::runtime_error("RPA1 entry path exceeds the path budget"); + } + if (path.find_first_of(std::string_view("\0:", 2)) != std::string_view::npos) { + throw std::runtime_error("RPA1 entry path contains a forbidden character"); + } + if (path.find_first_of("/\\") == 0) { + throw std::runtime_error("RPA1 entry path is absolute"); + } + + std::size_t begin = 0; + for (;;) { + const auto end = path.find_first_of("/\\", begin); + const auto segment = path.substr(begin, end == std::string_view::npos ? end : end - begin); + if (segment == "..") { + throw std::runtime_error("RPA1 entry path contains parent traversal"); + } + if (end == std::string_view::npos) { + return; + } + begin = end + 1; + } + } + } // namespace + + std::vector parse_rpa1_index(const std::span compressed_index, + const std::int64_t archive_size, const rpa1_limits &limits) + { + if (archive_size < 0) { + throw std::runtime_error("RPA1 data file size is negative"); + } + if (compressed_index.size() > limits.max_compressed_index_bytes) { + throw std::runtime_error("RPA1 compressed index exceeds the metadata budget"); + } + const auto decompressed = compression::decompress_zlib(compressed_index, limits.max_decompressed_index_bytes); + const auto root = load_index(decompressed); + const auto &dict = root->as_dict(); + if (dict.size() > limits.max_entries) { + throw std::runtime_error("RPA1 index exceeds the entry budget"); + } + + std::vector result; + result.reserve(dict.size()); + std::size_t fragment_count = 0; + for (const auto &[path, value] : dict) { + validate_path(path, limits.max_path_bytes); + const auto &fragments = value->as_list(); + if (fragments.empty()) { + throw std::runtime_error("RPA1 entry has no fragments"); + } + if (fragments.size() > limits.max_fragments - fragment_count) { + throw std::runtime_error("RPA1 index exceeds the fragment budget"); + } + fragment_count += fragments.size(); + + rpa1_entry entry{path, {}, 0}; + entry.fragments.reserve(fragments.size()); + for (const auto &fragment : fragments) { + const auto &tuple = fragment->as_tuple(); + if (tuple.size() != 2) { + throw std::runtime_error("RPA1 fragment must contain exactly two integers"); + } + const auto offset = tuple[0]->as_int64(); + const auto size = tuple[1]->as_int64(); + if (offset < 0 || size < 0) { + throw std::runtime_error("RPA1 fragment has a negative extent"); + } + if (offset > archive_size || size > archive_size - offset) { + throw std::runtime_error("RPA1 fragment exceeds the data file"); + } + if (size > std::numeric_limits::max() - entry.size) { + throw std::runtime_error("RPA1 entry size overflows int64"); + } + entry.size += size; + entry.fragments.push_back({offset, size}); + } + result.push_back(std::move(entry)); + } + return result; + } +} // namespace unpackrat::renpy diff --git a/src/modules/renpy/rpa1_index.h b/src/modules/renpy/rpa1_index.h new file mode 100644 index 0000000..188287c --- /dev/null +++ b/src/modules/renpy/rpa1_index.h @@ -0,0 +1,43 @@ +#ifndef UNPACKRAT_MODULES_RENPY_RPA1_INDEX_H +#define UNPACKRAT_MODULES_RENPY_RPA1_INDEX_H + +#include "../../core/archive_limits.h" + +#include +#include +#include +#include +#include + +namespace unpackrat::renpy +{ + struct rpa1_fragment final + { + std::int64_t offset; + std::int64_t size; + }; + + struct rpa1_entry final + { + std::string path; + std::vector fragments; + std::int64_t size = 0; + }; + + struct rpa1_limits final + { + std::size_t max_compressed_index_bytes = std::size_t{64} * 1024 * 1024; + std::size_t max_decompressed_index_bytes = std::size_t{64} * 1024 * 1024; + std::size_t max_entries = archive_limits::max_entry_count; + std::size_t max_fragments = 1'000'000; + std::size_t max_path_bytes = archive_limits::max_path_bytes; + }; + + // RPA1 stores its zlib-compressed Pickle index separately from the headerless data file. + // Each path maps to an ordered, nonempty list of (offset, size) fragments. The caller + // supplies the data file's authoritative size; all returned extents fit that file. + [[nodiscard]] std::vector parse_rpa1_index(std::span compressed_index, + std::int64_t archive_size, const rpa1_limits &limits = {}); +} // namespace unpackrat::renpy + +#endif diff --git a/src/tests/data/renpy/README.md b/src/tests/data/renpy/README.md index 0adbcbc..a5f5eec 100644 --- a/src/tests/data/renpy/README.md +++ b/src/tests/data/renpy/README.md @@ -1,4 +1,6 @@ -# Oversized Ren'Py index fixture +# Ren'Py archive fixtures + +## Oversized index fixture `expanded-index-over-budget.rpa` is a 65,271-byte RPA-2.0 archive. Its index begins at offset 32 and is a zlib stream that expands to exactly 67,108,865 `N` bytes: @@ -21,3 +23,36 @@ header = b"RPA-2.0 " + format(32, "016x").encode("ascii") + b"\n" archive = header.ljust(32, b"\0") + zlib.compress(b"N" * expanded_size, 9) Path("expanded-index-over-budget.rpa").write_bytes(archive) ``` + +## RPA-1.0 split-file pair + +`rpa1-split.rpa` and `rpa1-split.rpi` are authored test data, unrelated to any +game assets. The 29-byte data file contains padding and three fragments of +`dir/hello.txt`; the index lists the fragments in logical order, deliberately +different from their physical order. Extracting that entry must produce exactly +the 18 ASCII bytes `first:second:third`. The second entry, `empty.txt`, has a +zero-length range at the end of the data file and must extract as an empty file. +The `.rpa` file is the archive entry point; its neighboring `.rpi` file is a +required index sidecar and must not open as an archive by itself. + +The 85-byte index is a zlib-compressed protocol-2 Pickle dictionary. Regenerate +both files from this directory with Python's standard library: + +```python +from pathlib import Path +import pickle +import zlib + +data = b"PAD!third\xfffirst:\x00\x01second:TAIL" +index = { + "dir/hello.txt": [(10, 6), (18, 7), (4, 5)], + "empty.txt": [(len(data), 0)], +} +Path("rpa1-split.rpa").write_bytes(data) +Path("rpa1-split.rpi").write_bytes(zlib.compress(pickle.dumps(index, protocol=2), 9)) +``` + +SHA-256 values of the checked-in files: + +- `rpa1-split.rpa`: `cc75c66d110b50aa52eed163d72bae23953e2ac0a81a0856a5a0f06a2931ea68`. +- `rpa1-split.rpi`: `03fd8fb47ed04b503141d4520a564c269eb1ff1fcedf1b88980ff6b94c172bef`. diff --git a/src/tests/data/renpy/rpa1-split.rpa b/src/tests/data/renpy/rpa1-split.rpa new file mode 100644 index 0000000000000000000000000000000000000000..588dc5c7d51c0bd7c500a8ca88c2dee78f8e8173 GIT binary patch literal 29 kcmWG=bWtqH$Sg|vpO#rvTw=w*Se%-ipO<14;^^rE0H8YxQ~&?~ literal 0 HcmV?d00001 diff --git a/src/tests/data/renpy/rpa1-split.rpi b/src/tests/data/renpy/rpa1-split.rpi new file mode 100644 index 0000000000000000000000000000000000000000..820081f0d8131fd3af3e359bad4d00ba204f60e1 GIT binary patch literal 85 zcmV-b0IL6Z+G}8{Eo9J$;ALQ7NXabH&q&S5$=55XC@Ex&Eo9R0=JIB1D`fT-@@8)< rWbtP4W^F5EO^x6Ls!q) #include @@ -710,6 +712,245 @@ namespace test std::filesystem::remove(invalid_archive_path, error); } + namespace + { + class rpa1_pair final + { + public: + rpa1_pair() + { + const auto fixtures = std::filesystem::path(__FILE__).parent_path().parent_path() / "data/renpy"; + std::filesystem::copy_file(fixtures / "rpa1-split.rpa", data); + std::filesystem::copy_file(fixtures / "rpa1-split.rpi", index); + } + + support::temporary_directory directory{"rpa1"}; + std::filesystem::path data = directory.path() / L"архив.extra.rpa"; + std::filesystem::path index = directory.path() / L"архив.extra.rpi"; + }; + + void write_rpa1_bytes(const std::filesystem::path &path, const std::span bytes) + { + std::ofstream output(path, std::ios::binary); + REQUIRE(output.is_open()); + output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + REQUIRE(output.good()); + } + + void write_rpa1_pickle(const std::filesystem::path &path, const std::string_view pickle) + { + write_rpa1_bytes(path, support::compress_zlib_fixture(std::as_bytes(std::span(pickle)))); + } + } // namespace + + TEST_CASE("module ABI: RPA1 requires the exact adjacent index and data extension", "[contract][rpa1]") + { + const auto scenario = GENERATE(0, 1, 2, 3); + CAPTURE(scenario); + rpa1_pair pair; + if (scenario == 0) { + REQUIRE(std::filesystem::remove(pair.index)); + } else if (scenario == 1) { + std::filesystem::rename(pair.index, pair.directory.path() / "other.rpi"); + } else if (scenario == 2) { + auto renamed = pair.data; + renamed.replace_extension(".bin"); + std::filesystem::rename(pair.data, renamed); + pair.data = renamed; + } else { + pair.data = pair.index; + } + c_module loaded("renpy.so"); + CHECK(loaded.open(pair.data) == false); + } + + TEST_CASE("module ABI: RPA1 rejects unreadable and corrupt sidecars while opening", "[contract][rpa1]") + { + const auto scenario = GENERATE(0, 1, 2, 3, 4, 5, 6); + CAPTURE(scenario); + rpa1_pair pair; + if (scenario == 0) { + write_rpa1_bytes(pair.index, {}); + } else if (scenario == 1) { + std::filesystem::resize_file(pair.index, 5); + } else if (scenario == 2) { + write_rpa1_pickle(pair.index, "N."); + } else if (scenario == 3) { + std::filesystem::resize_file(pair.index, 64ULL * 1024 * 1024 + 1); + } else if (scenario == 4) { + REQUIRE(std::filesystem::remove(pair.index)); + REQUIRE(std::filesystem::create_directory(pair.index)); + } else { + write_rpa1_pickle(pair.index, scenario == 5 ? "Ibad\n." : "I999999999999999999999999999999\n."); + } + c_module loaded("renpy.so"); + CHECK(loaded.open(pair.data) == false); + } + + TEST_CASE("module ABI: RPA1 snapshots its index and accepts uppercase data suffixes", "[contract][rpa1]") + { + rpa1_pair pair; + auto renamed = pair.data; + renamed.replace_extension(".RPA"); + std::filesystem::rename(pair.data, renamed); + c_module loaded("renpy.so"); + REQUIRE(loaded.open(renamed)); + REQUIRE(std::filesystem::remove(pair.index)); + const auto files = loaded.list_files(); + REQUIRE(files.size() == 2); + const auto entry = std::ranges::find_if(files, [](const file &item) { return item.path == L"dir\\hello.txt"; }); + REQUIRE(entry != files.end()); + const temporary_output_file output(pair.directory.path() / "output.txt"); + entry->extract(output.path()); + std::ifstream input(output.path(), std::ios::binary); + const std::string contents{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + CHECK(contents == "first:second:third"); + CHECK(entry->uncompressed_size == 18); + CHECK(entry->compressed_size == 18); + } + + TEST_CASE("module ABI: RPA1 permits an empty archive and zero byte data", "[contract][rpa1]") + { + rpa1_pair pair; + write_rpa1_bytes(pair.data, {}); + write_rpa1_pickle(pair.index, "}."); + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + CHECK(loaded.list_files().empty()); + } + + TEST_CASE("module ABI: RPA1 streams large fragments and reports exact progress", "[contract][rpa1]") + { + rpa1_pair pair; + const std::string contents(128 * 1024 + 7, 'x'); + write_rpa1_bytes(pair.data, std::as_bytes(std::span(contents))); + std::string pickle = "}U"; + pickle.push_back(12); + pickle.append("файл.txt]I0\nI131079\n") + .append("\x86" + "as."); + write_rpa1_pickle(pair.index, pickle); + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + const auto files = loaded.list_files(); + REQUIRE(files.size() == 1); + CHECK(files.front().path == L"файл.txt"); + std::int64_t progress = 0; + const temporary_output_file output(pair.directory.path() / "output.txt"); + const auto callback = [](void *context, const int64_t amount) -> int { + *static_cast(context) += amount; + return TRUE; + }; + CHECK(loaded.extract_file_status(0, output.path(), callback, &progress) == SER_SUCCESS); + CHECK(progress == static_cast(contents.size())); + std::ifstream input(output.path(), std::ios::binary); + const std::string actual{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + CHECK(actual == contents); + } + + TEST_CASE("module ABI: RPA1 extraction cleans partial output on source and callback failures", "[contract][rpa1]") + { + const auto scenario = GENERATE(0, 1, 2); + CAPTURE(scenario); + rpa1_pair pair; + if (scenario == 0) { + const std::string body(std::size_t{256} * 1024, 'x'); + write_rpa1_bytes(pair.data, std::as_bytes(std::span(body))); + std::string pickle = "}U"; + pickle.push_back(13); + pickle.append("dir/hello.txt]I131072\nI6\n") + .append("\x86" + "as."); + write_rpa1_pickle(pair.index, pickle); + } + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + const auto files = loaded.list_files(); + const auto entry = std::ranges::find_if(files, [](const file &item) { return item.path == L"dir\\hello.txt"; }); + REQUIRE(entry != files.end()); + const auto item_index = static_cast(std::distance(files.begin(), entry)); + const temporary_output_file output(pair.directory.path() / "output.txt"); + const auto callback = [](void *context, int64_t) -> int { + if (*static_cast(context) == 2) { + throw std::runtime_error("RPA1 callback failure"); + } + return *static_cast(context) == 0 ? TRUE : FALSE; + }; + if (scenario == 0) { + const locked_source_range locked(pair.data, 128ULL * 1024, 6); + CHECK(loaded.extract_file_status(item_index, output.path(), callback, const_cast(&scenario)) == + SER_ERROR_READ); + } else { + const auto expected = scenario == 1 ? SER_USERABORT : SER_ERROR_SYSTEM; + CHECK(loaded.extract_file_status(item_index, output.path(), callback, const_cast(&scenario)) == + expected); + } + CHECK(std::filesystem::exists(output.path()) == false); + } + + TEST_CASE("module ABI: modern RPA ignores an adjacent RPA1 index", "[contract][rpa1]") + { + const auto sidecar = GENERATE(false, true); + rpa1_pair pair; + const auto contents = support::make_renpy_archive("modern.txt", "modern payload"); + write_rpa1_bytes(pair.data, std::as_bytes(std::span(contents))); + if (sidecar) { + write_rpa1_bytes(pair.index, {}); + } else { + REQUIRE(std::filesystem::remove(pair.index)); + } + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + const auto files = loaded.list_files(); + REQUIRE(files.size() == 1); + CHECK(files.front().path == L"modern.txt"); + } + + TEST_CASE("module ABI: RPA1 payload may begin with a modern signature", "[contract][rpa1][rpa1-collision]") + { + rpa1_pair pair; + const auto contents = + GENERATE(std::string("RPA-payload"), std::string("RPA-2.0 0000000000000000\n"), + std::string("RPA-2.0 0000000000000020\n"), [] { + const auto bytes = support::make_renpy_archive_with_index( + support::byte_buffer{'I', 'b', 'a', 'd', '\n', '.'}); + return std::string(reinterpret_cast(bytes.data()), bytes.size()); + }()); + write_rpa1_bytes(pair.data, std::as_bytes(std::span(contents))); + std::string pickle = "}U"; + pickle.push_back(1); + pickle.append("x]I0\nI") + .append(std::to_string(contents.size())) + .append("\n\x86" + "as."); + write_rpa1_pickle(pair.index, pickle); + c_module loaded("renpy.so"); + REQUIRE(loaded.open(pair.data)); + const auto files = loaded.list_files(); + REQUIRE(files.size() == 1); + const temporary_output_file output(pair.directory.path() / "output.txt"); + files.front().extract(output.path()); + std::ifstream input(output.path(), std::ios::binary); + const std::string actual{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + CHECK(actual == contents); + } + + TEST_CASE("module ABI: modern RPA rejects a truncated supplied signature", "[contract][rpa1]") + { + const support::temporary_archive archive("modern-short-hint", support::make_renpy_archive("x", "body")); + c_module loaded("renpy.so"); + const char prefix = 'R'; + const StorageOpenParams params{.StructSize = sizeof(StorageOpenParams), + .FilePath = archive.path().c_str(), + .Password = nullptr, + .Data = &prefix, + .DataSize = 1}; + HANDLE storage = nullptr; + StorageGeneralInfo info{}; + CHECK(loaded.api().OpenStorage(params, &storage, &info) == SOR_INVALID_FILE); + CHECK(storage == nullptr); + } + TEST_CASE("module ABI: malformed RenPy index shapes are rejected", "[contract]") { const auto expect_rejected = [](const std::string_view label, const support::byte_buffer &pickle_index) { diff --git a/src/tests/integration/archives.cpp b/src/tests/integration/archives.cpp index eef1a22..363fc48 100644 --- a/src/tests/integration/archives.cpp +++ b/src/tests/integration/archives.cpp @@ -9,6 +9,12 @@ #include +TEST_CASE("archives: hermetic RenPy RPA 1.0 sidecar", "[integration][hermetic][rpa1]") +{ + const auto archive = std::filesystem::path(__FILE__).parent_path().parent_path() / "data/renpy/rpa1-split.rpa"; + test::test_archive(archive, {{L"dir\\hello.txt", "first:second:third"}, {L"empty.txt", ""}}); +} + TEST_CASE("archives: hermetic RenPy RPA 2.0", "[integration][hermetic]") { const auto contents = test::support::make_renpy_archive("dir/hello.txt", "renpy payload"); diff --git a/src/tests/unit/pickle.cpp b/src/tests/unit/pickle.cpp index cbc7751..77328d5 100644 --- a/src/tests/unit/pickle.cpp +++ b/src/tests/unit/pickle.cpp @@ -108,8 +108,146 @@ namespace constexpr std::array container_factories{ list_container, tuple_container, dict_container}; + + std::string doubling_memo(const std::size_t depth) + { + std::string result("Nq\0", 3); + for (std::size_t level = 0; level < depth; ++level) { + result.append("h\0", 2); + result.push_back('\x86'); + result.append("q\0", 2); + } + result.push_back('.'); + return result; + } } // namespace +TEST_CASE("pickle: memo amplification is rejected before ownership grows without bound", "[pickle-budget]") +{ + // Each five-byte round doubles the owned subtree without approaching the depth limit. + // The complete legacy parse is finite (about one million live nodes) so this regression + // can demonstrate the missing check without deliberately exhausting the process. + REQUIRE_THROWS_WITH(pickle::loads(doubling_memo(18)), "Pickle node budget exceeded"); +} + +TEST_CASE("pickle: every parsed value is charged to the exact node boundary", "[pickle-budget]") +{ + struct node_case final + { + std::string bytes; + std::size_t nodes; + pickle::value::type type; + }; + const std::vector cases{ + {"N.", 1, pickle::value::type::none}, + {std::string("\x88.", 2), 1, pickle::value::type::bool_}, + {std::string("\x89.", 2), 1, pickle::value::type::bool_}, + {std::string("J\x01\0\0\0.", 6), 1, pickle::value::type::int64}, + {std::string("K\x01.", 3), 1, pickle::value::type::int64}, + {std::string("M\x01\0.", 4), 1, pickle::value::type::int64}, + {"I42\n.", 1, pickle::value::type::int64}, + {std::string("G\x3f\xf0\0\0\0\0\0\0.", 10), 1, pickle::value::type::float64}, + {std::string("U\x01s.", 4), 1, pickle::value::type::string}, + {std::string("T\x01\0\0\0s.", 7), 1, pickle::value::type::string}, + {std::string("\x8c\x01s.", 4), 1, pickle::value::type::string}, + {std::string("X\x01\0\0\0s.", 7), 1, pickle::value::type::string}, + {std::string("C\x01s.", 4), 1, pickle::value::type::bytes}, + {std::string("B\x01\0\0\0s.", 7), 1, pickle::value::type::bytes}, + {"].", 1, pickle::value::type::list}, + {"(Nl.", 2, pickle::value::type::list}, + {").", 1, pickle::value::type::tuple}, + {"(Nt.", 2, pickle::value::type::tuple}, + {std::string("N\x85.", 3), 2, pickle::value::type::tuple}, + {std::string("NN\x86.", 4), 3, pickle::value::type::tuple}, + {std::string("NNN\x87.", 5), 4, pickle::value::type::tuple}, + {"}.", 1, pickle::value::type::dict}, + {std::string("(U\x01kNd.", 7), 3, pickle::value::type::dict}, + {std::string("\x8a\0.", 3), 1, pickle::value::type::int64}, + {std::string("\x8a\x01\x07.", 4), 1, pickle::value::type::int64}, + }; + for (const auto &item : cases) { + CAPTURE(item.bytes, item.nodes); + pickle::limits limits; + limits.max_nodes = item.nodes; + REQUIRE(pickle::loads(item.bytes, limits)->get_type() == item.type); + --limits.max_nodes; + REQUIRE_THROWS_WITH(pickle::loads(item.bytes, limits), "Pickle node budget exceeded"); + } +} + +TEST_CASE("pickle: all memo opcodes share the parsing node budget", "[pickle-budget]") +{ + const std::array stores{ + {std::string("q\0", 2), std::string("r\0\0\0\0", 5), std::string(1, '\x94')}}; + const std::array gets{{std::string("h\0", 2), std::string("j\0\0\0\0", 5)}}; + for (const auto &store : stores) { + for (const auto &get : gets) { + std::string bytes = "N"; + bytes.append(store); + bytes.append(get); + bytes.append("\x86.", 2); + pickle::limits limits; + limits.max_nodes = 4; + REQUIRE(pickle::loads(bytes, limits)->as_tuple().size() == 2); + for (const auto budget : {std::size_t{1}, std::size_t{2}, std::size_t{3}}) { + CAPTURE(budget); + limits.max_nodes = budget; + REQUIRE_THROWS_WITH(pickle::loads(bytes, limits), "Pickle node budget exceeded"); + } + } + } + pickle::limits limits; + limits.max_nodes = 10; + REQUIRE_THROWS_WITH(pickle::loads(doubling_memo(3), limits), "Pickle node budget exceeded"); +} + +TEST_CASE("pickle: owned string and dictionary key copies have an aggregate byte budget", "[pickle-budget]") +{ + const std::array cases{std::string("U\x03" + "foo.", + 6), + std::string("C\x03" + "foo.", + 6), + std::string("I123\n.", 6), + std::string("(U\x01kNd.", 7), + std::string("}U\x01kNs.", 7), + std::string("}(U\x01kNu.", 8), + std::string("U\x01sq\0h\0\x86.", 9)}; + constexpr std::array required_bytes{3, 3, 3, 2, 2, 2, 3}; + for (std::size_t index = 0; index < cases.size(); ++index) { + CAPTURE(index); + pickle::limits limits; + limits.max_owned_bytes = required_bytes[index]; + REQUIRE(pickle::loads(cases[index], limits) != nullptr); + --limits.max_owned_bytes; + REQUIRE_THROWS_WITH(pickle::loads(cases[index], limits), "Pickle owned byte budget exceeded"); + } + const std::string no_bytes = "N."; + pickle::limits zero_bytes; + zero_bytes.max_owned_bytes = 0; + REQUIRE(pickle::loads(no_bytes, zero_bytes)->get_type() == pickle::value::type::none); + const auto span = std::as_bytes(std::span(cases[0])); + REQUIRE_THROWS_WITH(pickle::loads(span, zero_bytes), "Pickle owned byte budget exceeded"); +} + +TEST_CASE("pickle: recursive public clones share exact node and owned byte budgets", "[pickle-budget]") +{ + const auto tree = pickle::loads(std::string("}U\x01k]U\x02hi" + "as.")); + pickle::limits limits; + limits.max_nodes = 3; + limits.max_owned_bytes = 3; + const auto copy = pickle::clone(*tree, limits); + REQUIRE(copy->as_dict().at("k")->as_list()[0]->as_string() == "hi"); + REQUIRE(copy->as_dict().at("k").get() != tree->as_dict().at("k").get()); + limits.max_nodes = 2; + REQUIRE_THROWS_WITH(pickle::clone(*tree, limits), "Pickle node budget exceeded"); + limits.max_nodes = 3; + limits.max_owned_bytes = 2; + REQUIRE_THROWS_WITH(pickle::clone(*tree, limits), "Pickle owned byte budget exceeded"); +} + TEST_CASE("pickle: container nesting is bounded before ownership grows") { for (const auto &encoding : container_encodings) { diff --git a/src/tests/unit/renpy_rpa1_index.cpp b/src/tests/unit/renpy_rpa1_index.cpp new file mode 100644 index 0000000..ec920b5 --- /dev/null +++ b/src/tests/unit/renpy_rpa1_index.cpp @@ -0,0 +1,245 @@ +#include "../../modules/renpy/rpa1_index.h" +#include "../support/zlib_fixture.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace +{ + using unpackrat::renpy::parse_rpa1_index; + using unpackrat::renpy::rpa1_fragment; + using unpackrat::renpy::rpa1_limits; + + struct fixture_entry final + { + std::string path; + std::vector fragments; + }; + + void append_string(std::string &pickle, const std::string_view value) + { + pickle.push_back('X'); + const auto size = static_cast(value.size()); + for (unsigned shift = 0; shift < 32; shift += 8) { + pickle.push_back(static_cast((size >> shift) & 0xffU)); + } + pickle.append(value); + } + + std::string raw_index(const std::vector &entries) + { + std::string pickle = "}"; + for (const auto &entry : entries) { + append_string(pickle, entry.path); + pickle.push_back(']'); + for (const auto &fragment : entry.fragments) { + pickle.append("I").append(std::to_string(fragment.offset)).append("\n"); + pickle.append("I").append(std::to_string(fragment.size)).append("\n"); + pickle.push_back(static_cast(0x86)); + pickle.push_back('a'); + } + pickle.push_back('s'); + } + pickle.push_back('.'); + return pickle; + } + + std::vector compressed_pickle(const std::string_view pickle) + { + return test::support::compress_zlib_fixture(std::as_bytes(std::span(pickle.data(), pickle.size()))); + } + + std::vector index(const std::vector &entries) + { + return compressed_pickle(raw_index(entries)); + } + + std::vector malformed_value(const std::string_view value) + { + std::string pickle = "}"; + append_string(pickle, "file"); + pickle.append(value).append("s."); + return compressed_pickle(pickle); + } +} // namespace + +TEST_CASE("RenPy RPA1 index: fragments retain their order and logical sizes", "[renpy][rpa1]") +{ + const auto result = + parse_rpa1_index(index({{"dir/hello.txt", {{8, 2}, {0, 3}, {10, 0}}}, {"other\\file.bin", {{3, 5}}}}), 10); + REQUIRE(result.size() == 2); + const auto found = std::ranges::find(result, "dir/hello.txt", &unpackrat::renpy::rpa1_entry::path); + REQUIRE(found != result.end()); + CHECK(found->size == 5); + REQUIRE(found->fragments.size() == 3); + CHECK(found->fragments[0].offset == 8); + CHECK(found->fragments[0].size == 2); + CHECK(found->fragments[1].offset == 0); + CHECK(found->fragments[1].size == 3); + CHECK(found->fragments[2].offset == 10); + CHECK(found->fragments[2].size == 0); + const auto other = std::ranges::find(result, "other\\file.bin", &unpackrat::renpy::rpa1_entry::path); + REQUIRE(other != result.end()); + CHECK(other->size == 5); +} + +TEST_CASE("RenPy RPA1 index: empty archives and empty entries are valid", "[renpy][rpa1]") +{ + CHECK(parse_rpa1_index(index({}), 0).empty()); + const auto result = parse_rpa1_index(index({{"empty", {{0, 0}}}}), 0); + REQUIRE(result.size() == 1); + CHECK(result[0].path == "empty"); + CHECK(result[0].size == 0); + REQUIRE(result[0].fragments.size() == 1); + CHECK(result[0].fragments[0].offset == 0); + CHECK(result[0].fragments[0].size == 0); + CHECK_THROWS_AS(parse_rpa1_index(index({}), -1), std::runtime_error); +} + +TEST_CASE("RenPy RPA1 index: compressed and expanded budgets have exact boundaries", "[renpy][rpa1]") +{ + const std::vector entries{{"file", {{0, 1}}}}; + const auto pickle = raw_index(entries); + const auto compressed = compressed_pickle(pickle); + rpa1_limits limits; + limits.max_compressed_index_bytes = compressed.size(); + limits.max_decompressed_index_bytes = pickle.size(); + const auto accepted = parse_rpa1_index(compressed, 1, limits); + REQUIRE(accepted.size() == 1); + CHECK(accepted[0].size == 1); + + --limits.max_compressed_index_bytes; + CHECK_THROWS_AS(parse_rpa1_index(compressed, 1, limits), std::runtime_error); + ++limits.max_compressed_index_bytes; + --limits.max_decompressed_index_bytes; + CHECK_THROWS_AS(parse_rpa1_index(compressed, 1, limits), std::runtime_error); +} + +TEST_CASE("RenPy RPA1 index: corrupt compressed data and malformed pickle fail closed", "[renpy][rpa1]") +{ + CHECK_THROWS_AS(parse_rpa1_index({}, 1), std::runtime_error); + const std::array invalid{std::byte{'n'}, std::byte{'o'}}; + CHECK_THROWS_AS(parse_rpa1_index(invalid, 1), std::runtime_error); + auto truncated = index({{"file", {{0, 1}}}}); + truncated.pop_back(); + CHECK_THROWS_AS(parse_rpa1_index(truncated, 1), std::runtime_error); + auto trailing = index({{"file", {{0, 1}}}}); + trailing.push_back(std::byte{0}); + CHECK_THROWS_AS(parse_rpa1_index(trailing, 1), std::runtime_error); + CHECK_THROWS_AS(parse_rpa1_index(compressed_pickle("N."), 1), std::runtime_error); + CHECK_THROWS_AS(parse_rpa1_index(compressed_pickle("?"), 1), std::runtime_error); + CHECK_THROWS_AS(parse_rpa1_index(compressed_pickle("Ibad\n."), 1), std::runtime_error); + CHECK_THROWS_AS(parse_rpa1_index(compressed_pickle("I999999999999999999999999999999\n."), 1), std::runtime_error); +} + +TEST_CASE("RenPy RPA1 index: property lists require nonempty integer pairs", "[renpy][rpa1]") +{ + const std::array invalid_values{ + "N", "]", "]Na", "])a", "](I0\nta", "](I0\nI1\nI2\nta", "](NI1\nta", "](I0\nNta"}; + for (const auto value : invalid_values) { + CAPTURE(value); + CHECK_THROWS_AS(parse_rpa1_index(malformed_value(value), 10), std::runtime_error); + } +} + +TEST_CASE("RenPy RPA1 index: entry and aggregate fragment limits are enforced", "[renpy][rpa1]") +{ + const auto two_entries = index({{"first", {{0, 1}}}, {"second", {{1, 1}}}}); + rpa1_limits limits; + limits.max_entries = 2; + limits.max_fragments = 2; + CHECK(parse_rpa1_index(two_entries, 2, limits).size() == 2); + limits.max_entries = 1; + CHECK_THROWS_AS(parse_rpa1_index(two_entries, 2, limits), std::runtime_error); + limits.max_entries = 2; + limits.max_fragments = 1; + CHECK_THROWS_AS(parse_rpa1_index(two_entries, 2, limits), std::runtime_error); + limits.max_fragments = 2; + CHECK_THROWS_AS(parse_rpa1_index(index({{"one", {{0, 1}, {1, 1}, {2, 1}}}}), 3, limits), std::runtime_error); + limits.max_entries = 0; + limits.max_fragments = 0; + CHECK(parse_rpa1_index(index({}), 0, limits).empty()); +} + +TEST_CASE("RenPy RPA1 index: range decision table rejects invalid extents", "[renpy][rpa1]") +{ + struct range_case final + { + std::int64_t offset; + std::int64_t size; + bool accepted; + }; + constexpr std::array cases{ + range_case{0, 0, true}, + range_case{0, 10, true}, + range_case{10, 0, true}, + range_case{9, 1, true}, + range_case{-1, 0, false}, + range_case{0, -1, false}, + range_case{11, 0, false}, + range_case{0, 11, false}, + range_case{9, 2, false}, + range_case{std::numeric_limits::max(), 1, false}, + range_case{1, std::numeric_limits::max(), false}, + }; + for (const auto &item : cases) { + CAPTURE(item.offset, item.size); + const auto compressed = index({{"file", {{item.offset, item.size}}}}); + if (item.accepted) { + const auto result = parse_rpa1_index(compressed, 10); + REQUIRE(result.size() == 1); + CHECK(result[0].size == item.size); + } else { + CHECK_THROWS_AS(parse_rpa1_index(compressed, 10), std::runtime_error); + } + } +} + +TEST_CASE("RenPy RPA1 index: fragment sums are checked before signed overflow", "[renpy][rpa1]") +{ + constexpr auto maximum = std::numeric_limits::max(); + const auto accepted = parse_rpa1_index(index({{"file", {{0, maximum - 1}, {0, 1}}}}), maximum); + REQUIRE(accepted.size() == 1); + CHECK(accepted[0].size == maximum); + CHECK_THROWS_AS(parse_rpa1_index(index({{"file", {{0, maximum}, {0, 1}}}}), maximum), std::runtime_error); +} + +TEST_CASE("RenPy RPA1 index: paths must remain relative and free of traversal", "[renpy][rpa1]") +{ + const std::vector rejected{ + "", "/root", "\\root", "C:file", "dir/file:stream", "..", + "../file", "dir/../file", "dir\\..\\file", "dir/..", "dir\\..", std::string("dir\0file", 8)}; + for (const auto &path : rejected) { + CAPTURE(path); + CHECK_THROWS_AS(parse_rpa1_index(index({{path, {{0, 0}}}}), 0), std::runtime_error); + } + const std::array accepted{"file", "dir/file", "dir\\file", "...", "a..b", "dir/./file"}; + for (const auto path : accepted) { + CAPTURE(path); + const auto result = parse_rpa1_index(index({{std::string(path), {{0, 0}}}}), 0); + REQUIRE(result.size() == 1); + CHECK(result[0].path == path); + } +} + +TEST_CASE("RenPy RPA1 index: path byte budgets preserve valid UTF8", "[renpy][rpa1]") +{ + const std::string unicode = "dir/\xd0\xbf\xd1\x83\xd1\x82\xd1\x8c.txt"; + rpa1_limits limits; + limits.max_path_bytes = unicode.size(); + const auto result = parse_rpa1_index(index({{unicode, {{0, 0}}}}), 0, limits); + REQUIRE(result.size() == 1); + CHECK(result[0].path == unicode); + --limits.max_path_bytes; + CHECK_THROWS_AS(parse_rpa1_index(index({{unicode, {{0, 0}}}}), 0, limits), std::runtime_error); +}