diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 99a8ce3320ad4e..582a00a32facff 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -305,6 +305,13 @@ def _extract_recreation_code(value): else: return 3 # NoneType + @staticmethod + def _encode_value(value): + if not isinstance(value, str): + return value + else: + return value.encode(_encoding) + def __init__(self, sequence=None, *, name=None): if name is None or sequence is not None: sequence = sequence or () @@ -312,7 +319,7 @@ def __init__(self, sequence=None, *, name=None): self._types_mapping[type(item)] if not isinstance(item, (str, bytes)) else self._types_mapping[type(item)] % ( - self._alignment * (len(item) // self._alignment + 1), + self._alignment * ((len(self._encode_value(item)) - 1) // self._alignment + 1), ) for item in sequence ] @@ -353,7 +360,7 @@ def __init__(self, sequence=None, *, name=None): "".join(_formats), self.shm.buf, self._offset_data_start, - *(v.encode(_enc) if isinstance(v, str) else v for v in sequence) + *(self._encode_value(v) for v in sequence) ) struct.pack_into( self._format_packing_metainfo, @@ -463,8 +470,7 @@ def __setitem__(self, position, value): else: allocated_length = self._allocated_offsets[position + 1] - item_offset - encoded_value = (value.encode(_encoding) - if isinstance(value, str) else value) + encoded_value = self._encode_value(value) if len(encoded_value) > allocated_length: raise ValueError("bytes/str item exceeds available storage") if current_format[-1] == "s": diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 36e0880bc08818..c5111355c1458a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4962,7 +4962,7 @@ def test_shared_memory_SharedMemoryManager_basics(self): def test_shared_memory_ShareableList_basics(self): sl = shared_memory.ShareableList( - ['howdy', b'HoWdY', -273.154, 100, None, True, 42] + ['howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥'] ) self.addCleanup(sl.shm.unlink) @@ -4972,11 +4972,11 @@ def test_shared_memory_ShareableList_basics(self): # Index Out of Range (get) with self.assertRaises(IndexError): - sl[7] + sl[8] # Index Out of Range (set) with self.assertRaises(IndexError): - sl[7] = 2 + sl[8] = 2 # Assign value without format change (str -> str) current_format = sl._get_packing_format(0) @@ -4984,10 +4984,10 @@ def test_shared_memory_ShareableList_basics(self): self.assertEqual(current_format, sl._get_packing_format(0)) # Verify attributes are readable. - self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q') + self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q16s') # Exercise len(). - self.assertEqual(len(sl), 7) + self.assertEqual(len(sl), 8) # Exercise index(). with warnings.catch_warnings(): @@ -4999,12 +4999,13 @@ def test_shared_memory_ShareableList_basics(self): # Exercise retrieving individual values. self.assertEqual(sl[0], 'howdy') - self.assertEqual(sl[-2], True) + self.assertEqual(sl[-3], True) + self.assertEqual(sl[-1], '💥 💥') # Exercise iterability. self.assertEqual( tuple(sl), - ('howdy', b'HoWdY', -273.154, 100, None, True, 42) + ('howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥') ) # Exercise modifying individual values. @@ -5012,17 +5013,17 @@ def test_shared_memory_ShareableList_basics(self): self.assertEqual(sl[3], 42) sl[4] = 'some' # Change type at a given position. self.assertEqual(sl[4], 'some') - self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q') + self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q16s') with self.assertRaisesRegex(ValueError, "exceeds available storage"): sl[4] = 'far too many' self.assertEqual(sl[4], 'some') - sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data - self.assertEqual(sl[0], 'encodés') + sl[0] = 'éncodé' # Exactly 8 bytes of UTF-8 data + self.assertEqual(sl[0], 'éncodé') self.assertEqual(sl[1], b'HoWdY') # no spillage with self.assertRaisesRegex(ValueError, "exceeds available storage"): - sl[0] = 'encodées' # Exactly 9 bytes of UTF-8 data + sl[0] = 'éncodés' # Exactly 9 bytes of UTF-8 data self.assertEqual(sl[1], b'HoWdY') with self.assertRaisesRegex(ValueError, "exceeds available storage"): diff --git a/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst new file mode 100644 index 00000000000000..db8366f6ef0f09 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst @@ -0,0 +1 @@ +Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList.