From c289854d96fcc5458e8a54283b02a195cb67363f Mon Sep 17 00:00:00 2001 From: junnplus Date: Mon, 24 May 2021 18:46:35 +0800 Subject: [PATCH 1/3] bpo-44170: Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList --- Lib/multiprocessing/shared_memory.py | 16 +++++++++++----- Lib/test/_test_multiprocessing.py | 24 ++++++++++++------------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 122b3fcebf3fed..44b4bf0964bc22 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -293,6 +293,13 @@ def _extract_recreation_code(value): else: return 3 # NoneType + @staticmethod + def _encode_value(value): + if not isinstance(value, str): + return value + else: + return value.encode(_encoding) + def __init__(self, sequence=None, *, name=None): if name is None or sequence is not None: sequence = sequence or () @@ -300,7 +307,7 @@ def __init__(self, sequence=None, *, name=None): self._types_mapping[type(item)] if not isinstance(item, (str, bytes)) else self._types_mapping[type(item)] % ( - self._alignment * (len(item) // self._alignment + 1), + self._alignment * (len(self._encode_value(item)) // self._alignment + 1), ) for item in sequence ] @@ -341,7 +348,7 @@ def __init__(self, sequence=None, *, name=None): "".join(_formats), self.shm.buf, self._offset_data_start, - *(v.encode(_enc) if isinstance(v, str) else v for v in sequence) + *(self._encode_value(v) for v in sequence) ) struct.pack_into( self._format_packing_metainfo, @@ -451,9 +458,8 @@ def __setitem__(self, position, value): else: allocated_length = self._allocated_offsets[position + 1] - item_offset - encoded_value = (value.encode(_encoding) - if isinstance(value, str) else value) - if len(encoded_value) > allocated_length: + encoded_value = self._encode_value(value) + if len(encoded_value) >= allocated_length: raise ValueError("bytes/str item exceeds available storage") if current_format[-1] == "s": new_format = current_format diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index ead92cfa2abfea..2105155d765332 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3997,7 +3997,7 @@ def test_shared_memory_SharedMemoryManager_basics(self): def test_shared_memory_ShareableList_basics(self): sl = shared_memory.ShareableList( - ['howdy', b'HoWdY', -273.154, 100, None, True, 42] + ['howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥'] ) self.addCleanup(sl.shm.unlink) @@ -4007,11 +4007,11 @@ def test_shared_memory_ShareableList_basics(self): # Index Out of Range (get) with self.assertRaises(IndexError): - sl[7] + sl[8] # Index Out of Range (set) with self.assertRaises(IndexError): - sl[7] = 2 + sl[8] = 2 # Assign value without format change (str -> str) current_format = sl._get_packing_format(0) @@ -4019,10 +4019,10 @@ def test_shared_memory_ShareableList_basics(self): self.assertEqual(current_format, sl._get_packing_format(0)) # Verify attributes are readable. - self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q') + self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q16s') # Exercise len(). - self.assertEqual(len(sl), 7) + self.assertEqual(len(sl), 8) # Exercise index(). with warnings.catch_warnings(): @@ -4034,12 +4034,13 @@ def test_shared_memory_ShareableList_basics(self): # Exercise retrieving individual values. self.assertEqual(sl[0], 'howdy') - self.assertEqual(sl[-2], True) + self.assertEqual(sl[-3], True) + self.assertEqual(sl[-1], '💥 💥') # Exercise iterability. self.assertEqual( tuple(sl), - ('howdy', b'HoWdY', -273.154, 100, None, True, 42) + ('howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥') ) # Exercise modifying individual values. @@ -4047,17 +4048,16 @@ def test_shared_memory_ShareableList_basics(self): self.assertEqual(sl[3], 42) sl[4] = 'some' # Change type at a given position. self.assertEqual(sl[4], 'some') - self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q') + self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q16s') with self.assertRaisesRegex(ValueError, "exceeds available storage"): sl[4] = 'far too many' self.assertEqual(sl[4], 'some') - sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data - self.assertEqual(sl[0], 'encodés') - self.assertEqual(sl[1], b'HoWdY') # no spillage + sl[0] = 'encodé' + self.assertEqual(sl[0], 'encodé') # no spillage with self.assertRaisesRegex(ValueError, "exceeds available storage"): - sl[0] = 'encodées' # Exactly 9 bytes of UTF-8 data + sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data self.assertEqual(sl[1], b'HoWdY') with self.assertRaisesRegex(ValueError, "exceeds available storage"): From 3431c1d74de00e72b69ff267b28d221708f64207 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 14:41:30 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst diff --git a/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst new file mode 100644 index 00000000000000..f74f9f2654f0f4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst @@ -0,0 +1 @@ +Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList. \ No newline at end of file From 9c8d18d530e3fd0683ab55e4aae02b4d38adf3d1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 12 Aug 2026 20:31:56 +0300 Subject: [PATCH 3/3] Round up the allocated size instead of rejecting the exact fit Allocating one extra alignment block for a value whose encoded size is a multiple of the alignment made a copy of the list wider than the original. Round the size up instead, so that it is the same for the same value, and keep allowing a value which fills the slot exactly: rejecting it broke setting an 8-byte string in a slot of an integer. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/multiprocessing/shared_memory.py | 4 ++-- Lib/test/_test_multiprocessing.py | 7 ++++--- .../next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 902b9ee16851d3..582a00a32facff 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -319,7 +319,7 @@ def __init__(self, sequence=None, *, name=None): self._types_mapping[type(item)] if not isinstance(item, (str, bytes)) else self._types_mapping[type(item)] % ( - self._alignment * (len(self._encode_value(item)) // self._alignment + 1), + self._alignment * ((len(self._encode_value(item)) - 1) // self._alignment + 1), ) for item in sequence ] @@ -471,7 +471,7 @@ def __setitem__(self, position, value): allocated_length = self._allocated_offsets[position + 1] - item_offset encoded_value = self._encode_value(value) - if len(encoded_value) >= allocated_length: + if len(encoded_value) > allocated_length: raise ValueError("bytes/str item exceeds available storage") if current_format[-1] == "s": new_format = current_format diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 226a5d3bee6c3e..c5111355c1458a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5018,11 +5018,12 @@ def test_shared_memory_ShareableList_basics(self): "exceeds available storage"): sl[4] = 'far too many' self.assertEqual(sl[4], 'some') - sl[0] = 'encodé' - self.assertEqual(sl[0], 'encodé') # no spillage + sl[0] = 'éncodé' # Exactly 8 bytes of UTF-8 data + self.assertEqual(sl[0], 'éncodé') + self.assertEqual(sl[1], b'HoWdY') # no spillage with self.assertRaisesRegex(ValueError, "exceeds available storage"): - sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data + sl[0] = 'éncodés' # Exactly 9 bytes of UTF-8 data self.assertEqual(sl[1], b'HoWdY') with self.assertRaisesRegex(ValueError, "exceeds available storage"): diff --git a/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst index f74f9f2654f0f4..db8366f6ef0f09 100644 --- a/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst +++ b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst @@ -1 +1 @@ -Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList. \ No newline at end of file +Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList.