From 368334bd045d368b5a2e46c0d197f9003720cada Mon Sep 17 00:00:00 2001 From: JotaJazzCL <181486793+JotaJazz@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:38:04 -0300 Subject: [PATCH 1/5] fslike: convert paths as str across the C++/Python boundary Since Python fslike paths are stored as str, the C++ callbacks still passed subpaths as bytes and read Directory.path and native paths with bytes(). Lookups in a Union never matched, so the engine could not find assets/converted and the game did not start. Let Cython convert std::string to and from str (UTF-8) in the fslike bridge, and read Python strings with str() in C++. Co-Authored-By: Claude Opus 5.5 --- libopenage/util/fslike/python.cpp | 2 +- libopenage/util/path.cpp | 2 +- libopenage/util/path.h | 6 ++++ openage/testing/testlist.py | 1 + openage/util/fslike/CMakeLists.txt | 1 + openage/util/fslike/cpp.pyx | 9 +++-- openage/util/fslike/cpp_tests.pyx | 55 ++++++++++++++++++++++++++++++ 7 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 openage/util/fslike/cpp_tests.pyx diff --git a/libopenage/util/fslike/python.cpp b/libopenage/util/fslike/python.cpp index 72e36ffbba..289692569f 100644 --- a/libopenage/util/fslike/python.cpp +++ b/libopenage/util/fslike/python.cpp @@ -67,7 +67,7 @@ std::string Python::get_native_path(const Path::subpath_t &subpath) { return ""; } else { - return ret.bytes(); + return ret.str(); } } diff --git a/libopenage/util/path.cpp b/libopenage/util/path.cpp index a625dec2ff..e03459e806 100644 --- a/libopenage/util/path.cpp +++ b/libopenage/util/path.cpp @@ -55,7 +55,7 @@ Path::Path( // test if fsobj is fslike.Directory if (fslike::pyx_fs_is_fslike_directory.call(fsobj_in.get_ref())) { this->fsobj = std::make_shared( - fsobj_in.getattr("path").bytes()); + fsobj_in.getattr("path").str()); } else { // we can't create a c++-variant of the path, diff --git a/libopenage/util/path.h b/libopenage/util/path.h index f74ba6e209..4a7d900fb9 100644 --- a/libopenage/util/path.h +++ b/libopenage/util/path.h @@ -7,6 +7,7 @@ */ +// pxd: from libcpp cimport bool // pxd: from libcpp.string cimport string #include // pxd: from libcpp.vector cimport vector @@ -42,6 +43,11 @@ class FSLike; * * FSLike *get_fsobj() except + * const vector[string] &get_subpath() except + + * + * bool is_file() except + + * bool is_dir() except + + * vector[string] list() except + + * string resolve_native_path(const string &mode) except + */ class OAAPI Path { public: diff --git a/openage/testing/testlist.py b/openage/testing/testlist.py index 7596146082..ad0659fd04 100644 --- a/openage/testing/testlist.py +++ b/openage/testing/testlist.py @@ -32,6 +32,7 @@ def tests_py(): ) yield ("openage.testing.misc_cpp.enum", "tests the interface for C++'s util::Enum class") yield ("openage.util.fslike.test.test", "test the filesystem abstraction subsystem") + yield ("openage.util.fslike.cpp_tests.test", "access Python fslikes through C++ paths") yield "openage.util.threading.test_concurrent_chain" diff --git a/openage/util/fslike/CMakeLists.txt b/openage/util/fslike/CMakeLists.txt index ee85d18c1a..d9210dc658 100644 --- a/openage/util/fslike/CMakeLists.txt +++ b/openage/util/fslike/CMakeLists.txt @@ -15,4 +15,5 @@ add_pxds( add_cython_modules( cpp.pyx + cpp_tests.pyx ) diff --git a/openage/util/fslike/cpp.pyx b/openage/util/fslike/cpp.pyx index 4182354d6a..7bd8d8231b 100644 --- a/openage/util/fslike/cpp.pyx +++ b/openage/util/fslike/cpp.pyx @@ -1,4 +1,5 @@ # Copyright 2017-2021 the openage authors. See copying.md for legal info. +# cython: c_string_type=unicode, c_string_encoding=utf8 """ Functions called from C++ to perform method calls on @@ -92,10 +93,8 @@ cdef class FSLikeCPPWrapper: def get_native_path(self, subpath): cdef string native_path = self.fsobj.get().get_native_path(subpath) - txt = bytes(native_path) - - if txt: - return txt + if not native_path.empty(): + return native_path else: return None @@ -284,7 +283,7 @@ cdef check_file_exists(object path, object fslike, const vector[string]& subpath raise FileNotFoundError("file could not be found in filesystem %s " "for path '%s'" % ( fslike, - b"/".join(subpath).decode(errors='ignore') + "/".join(subpath) )) diff --git a/openage/util/fslike/cpp_tests.pyx b/openage/util/fslike/cpp_tests.pyx new file mode 100644 index 0000000000..626244e106 --- /dev/null +++ b/openage/util/fslike/cpp_tests.pyx @@ -0,0 +1,55 @@ +# Copyright 2026-2026 the openage authors. See copying.md for legal info. +# cython: c_string_type=unicode, c_string_encoding=utf8 + +""" +Tests for accessing Python fslike objects from C++ through cpp.pyx. +""" + +import os +from tempfile import TemporaryDirectory + +from cpython.ref cimport PyObject + +from libopenage.pyinterface.pyobject cimport PyObj +from libopenage.util.path cimport Path as Path_cpp + +from ...testing.testing import assert_value +from .directory import Directory +from .union import Union +from .wrapper import WriteBlocker + + +def test(): + """ + Query Python fslikes through C++ paths, like the engine does with the + root union it gets from Python. + """ + cdef Path_cpp converted + cdef Path_cpp modpack_file + cdef Path_cpp blocked_file + + with TemporaryDirectory() as tmp: + os.makedirs(os.path.join(tmp, "converted")) + native_file = os.path.join(tmp, "converted", "modpack.toml") + with open(native_file, "w", encoding="utf-8"): + pass + + union = Union() + union.root["assets"].mount(Directory(tmp).root) + + converted = Path_cpp(["assets", "converted"], PyObj(union)) + assert_value(converted.is_dir(), True) + assert_value(converted.is_file(), False) + assert_value(list(converted.list()), ["modpack.toml"]) + + # resolving through the union yields a Directory, which C++ converts + # to its native fslike implementation + modpack_file = Path_cpp(["assets", "converted", "modpack.toml"], PyObj(union)) + assert_value(modpack_file.is_file(), True) + assert_value(modpack_file.resolve_native_path("r"), native_file) + + # the outer wrapper resolves to the inner one, which stays a Python + # fslike, so C++ asks Python for the native path + blocker = WriteBlocker(WriteBlocker(Directory(tmp).root).root) + blocked_file = Path_cpp(["converted", "modpack.toml"], PyObj(blocker)) + assert_value(blocked_file.resolve_native_path("r"), native_file) From df461b26309205213985c10e5b645081c5190538 Mon Sep 17 00:00:00 2001 From: JotaJazzCL <181486793+JotaJazz@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:38:04 -0300 Subject: [PATCH 2/5] gamestate: attach the renderer once the game is created The presenter attaches its renderer in its own thread, which can happen before GameSimulation::start() has created the game, because creating it loads all nyan data. That dereferenced a null game and crashed. Store the render factory and attach it to the game in start() if the game didn't exist yet. Co-Authored-By: Claude Opus 5.5 --- libopenage/gamestate/simulation.cpp | 10 +++++++++- libopenage/gamestate/simulation.h | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/libopenage/gamestate/simulation.cpp b/libopenage/gamestate/simulation.cpp index ed2627c95e..40c2c104b1 100644 --- a/libopenage/gamestate/simulation.cpp +++ b/libopenage/gamestate/simulation.cpp @@ -63,6 +63,11 @@ void GameSimulation::start() { this->entity_factory, this->terrain_factory); + // the presenter may have attached its renderer while the game was not created yet + if (this->render_factory) { + this->game->attach_renderer(this->render_factory); + } + this->running = true; log::log(MSG(info) << "Game simulation started"); @@ -117,7 +122,10 @@ const std::shared_ptr GameSimulation::get_commander void GameSimulation::attach_renderer(const std::shared_ptr &render_factory) { std::unique_lock lock{this->mutex}; - this->game->attach_renderer(render_factory); + this->render_factory = render_factory; + if (this->game) { + this->game->attach_renderer(render_factory); + } this->entity_factory->attach_renderer(render_factory); this->terrain_factory->attach_renderer(render_factory); } diff --git a/libopenage/gamestate/simulation.h b/libopenage/gamestate/simulation.h index b7e4331775..a6da76d093 100644 --- a/libopenage/gamestate/simulation.h +++ b/libopenage/gamestate/simulation.h @@ -125,7 +125,10 @@ class GameSimulation final { /** * Attach a renderer to the simulation. * - * @param factory Factory for creating render entities. + * May be called before the game is started; the game gets the renderer + * attached once it is created. + * + * @param render_factory Factory for creating render entities. */ void attach_renderer(const std::shared_ptr &render_factory); @@ -196,6 +199,11 @@ class GameSimulation final { // TODO: The game run by the engine std::shared_ptr game; + /** + * Factory for creating render entities, if a renderer is attached. + */ + std::shared_ptr render_factory; + /** * Mutex for thread-safe access to the simulation. */ From 0a3efaba65284b6bf87bbc436f6b4036e099c512 Mon Sep 17 00:00:00 2001 From: JotaJazzCL <181486793+JotaJazz@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:43:31 -0300 Subject: [PATCH 3/5] copying: add Nicolas Inostroza Co-Authored-By: Claude Opus 5.5 --- copying.md | 1 + 1 file changed, 1 insertion(+) diff --git a/copying.md b/copying.md index 1cf7a15546..cf5ff6fed7 100644 --- a/copying.md +++ b/copying.md @@ -174,6 +174,7 @@ _the openage authors_ are: | Erwan Billard | iacker | 82095453+iacker à users dawt noreply dawt github dawt com | | Deniz Kökden | denizkoekden | koekdendeniz à gmail dawt com | | Sergey Bronnikov | ligurio | estetus à gmail dawt com | +| Nicolas Inostroza | JotaJazz | 181486793+jotajazz à users dawt noreply dawt github dawt com | If you're a first-time committer, add yourself to the above list. This is not just for legal reasons, but also to keep an overview of all those nicknames. From f52b27dfe34b494bf5fde9fea70157aa1122b76d Mon Sep 17 00:00:00 2001 From: JotaJazzCL <181486793+JotaJazz@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:08:30 -0300 Subject: [PATCH 4/5] pathfinding: read sector size from the moved sectors in Grid The Grid constructor taking existing sectors moved the vector into the member and then read the sector size from the moved-from parameter, so it always threw std::out_of_range. Fixes #1828 Co-Authored-By: Claude Opus 5.5 --- libopenage/pathfinding/grid.cpp | 2 +- libopenage/pathfinding/tests.cpp | 15 +++++++++++++++ openage/testing/testlist.py | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/libopenage/pathfinding/grid.cpp b/libopenage/pathfinding/grid.cpp index 45427b009d..41dd9f4fdd 100644 --- a/libopenage/pathfinding/grid.cpp +++ b/libopenage/pathfinding/grid.cpp @@ -37,7 +37,7 @@ Grid::Grid(grid_id_t id, "Grid has size " << size[0] << "x" << size[1] << " (" << size[0] * size[1] << " sectors), " << "but only " << this->sectors.size() << " sectors were provided"); - this->sector_size = sectors.at(0)->get_cost_field()->get_size(); + this->sector_size = this->sectors.at(0)->get_cost_field()->get_size(); } grid_id_t Grid::get_id() const { diff --git a/libopenage/pathfinding/tests.cpp b/libopenage/pathfinding/tests.cpp index 2ac0960c34..87c6e036cb 100644 --- a/libopenage/pathfinding/tests.cpp +++ b/libopenage/pathfinding/tests.cpp @@ -7,8 +7,10 @@ #include "pathfinding/cost_field.h" #include "pathfinding/definitions.h" #include "pathfinding/flow_field.h" +#include "pathfinding/grid.h" #include "pathfinding/integration_field.h" #include "pathfinding/integrator.h" +#include "pathfinding/sector.h" #include "pathfinding/types.h" #include "time/time.h" @@ -113,6 +115,19 @@ void flow_field() { } } +void grid_from_sectors() { + std::vector> sectors; + for (size_t y = 0; y < 2; y++) { + for (size_t x = 0; x < 3; x++) { + sectors.push_back(std::make_shared(sectors.size(), coord::chunk(x, y), 5)); + } + } + + Grid grid{0, util::Vector2s{3, 2}, std::move(sectors)}; + TESTEQUALS(grid.get_sector_size(), 5); + TESTEQUALS(grid.get_sectors().size(), 6); +} + } // namespace tests } // namespace path diff --git a/openage/testing/testlist.py b/openage/testing/testlist.py index ad0659fd04..44c9fbb83d 100644 --- a/openage/testing/testlist.py +++ b/openage/testing/testlist.py @@ -80,6 +80,7 @@ def tests_cpp(): yield "openage::job::tests::test_job_manager" yield "openage::path::tests::path_node", "pathfinding" yield "openage::path::tests::flow_field", "pathfinding" + yield "openage::path::tests::grid_from_sectors", "pathfinding" yield "openage::pyinterface::tests::pyobject" yield "openage::pyinterface::tests::err_py_to_cpp" yield "openage::renderer::tests::font" From f158df015373fe3acfb89ddc91b1eed001d5e144 Mon Sep 17 00:00:00 2001 From: JotaJazzCL <181486793+JotaJazz@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:08:30 -0300 Subject: [PATCH 5/5] datastructure: let ConcurrentQueue::pop() wait with a single lock pop() locked the recursive mutex and then waited in front(), which locked it a second time. The wait only releases one lock level, so the mutex stayed held, push() could never add an element and pop() on an empty queue deadlocked. pop() now waits itself, holding only one lock. Fixes #1826 Co-Authored-By: Claude Opus 5.5 --- libopenage/datastructure/concurrent_queue.h | 25 +++++++++++++++------ libopenage/datastructure/tests.cpp | 18 +++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/libopenage/datastructure/concurrent_queue.h b/libopenage/datastructure/concurrent_queue.h index 84d5366e5a..c28d4c5142 100644 --- a/libopenage/datastructure/concurrent_queue.h +++ b/libopenage/datastructure/concurrent_queue.h @@ -40,9 +40,7 @@ class ConcurrentQueue { /** Returns the front item of the queue without removing it. */ T &front() { std::unique_lock lock{this->mutex}; - while (this->queue.empty()) { - this->elements_available.wait(lock); - } + this->wait_for_elements(lock); return this->queue.front(); } @@ -50,8 +48,9 @@ class ConcurrentQueue { template T pop([[maybe_unused]] typename std::enable_if_t and std::is_copy_constructible_v> *t = nullptr) { static_assert(sizeof...(None) == 0, "User-specified template arguments are prohibited."); - std::scoped_lock lock{this->mutex}; - T ret = this->front(); + std::unique_lock lock{this->mutex}; + this->wait_for_elements(lock); + T ret = this->queue.front(); this->queue.pop(); // Explicitly call the copy constructor since T cannot be move-constructed @@ -63,8 +62,9 @@ class ConcurrentQueue { template T pop([[maybe_unused]] typename std::enable_if_t> *t = nullptr) { static_assert(sizeof...(None) == 0, "User-specified template arguments are prohibited."); - std::scoped_lock lock{this->mutex}; - T ret = std::move(this->front()); + std::unique_lock lock{this->mutex}; + this->wait_for_elements(lock); + T ret = std::move(this->queue.front()); this->queue.pop(); return ret; } @@ -98,6 +98,17 @@ class ConcurrentQueue { } private: + /** + * Wait until the queue has elements. + * + * The wait only releases \p lock, so it must be the only lock + * this thread holds on the recursive mutex, otherwise producers + * can't push and the wait never ends. + */ + void wait_for_elements(std::unique_lock &lock) { + this->elements_available.wait(lock, [this] { return not this->queue.empty(); }); + } + /** The internally used queue. */ std::queue queue; diff --git a/libopenage/datastructure/tests.cpp b/libopenage/datastructure/tests.cpp index 40aa516951..285984a631 100644 --- a/libopenage/datastructure/tests.cpp +++ b/libopenage/datastructure/tests.cpp @@ -3,8 +3,10 @@ #include "tests.h" #include +#include #include #include +#include #include #include "testing/testing.h" @@ -354,11 +356,27 @@ void concurrent_queue_copy_move_elements_compilation() { res.reset_accumulated_construction_counter(); } +void concurrent_queue_pop_waits_for_push() { + openage::datastructure::ConcurrentQueue queue; + + int popped = 0; + std::thread consumer{[&] { popped = queue.pop(); }}; + + // the test passes either way, the delay just lets the consumer + // wait in pop() on the empty queue before the push + std::this_thread::sleep_for(std::chrono::milliseconds{10}); + queue.push(157); + consumer.join(); + + TESTEQUALS(popped, 157); +} + // exported test void concurrent_queue() { concurrent_queue_copy_only_elements_compilation(); concurrent_queue_move_only_elements_compilation(); concurrent_queue_copy_move_elements_compilation(); + concurrent_queue_pop_waits_for_push(); } } // namespace openage::datastructure::tests