From 3fd4b1da26dc91d7c33c8b2d414250b7b4c5bbaa Mon Sep 17 00:00:00 2001 From: getsentry-bot <10587625+getsentry-bot@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:02:13 +0000 Subject: [PATCH 1/7] ref: bump objectstore-client to 0.2.10 Co-Authored-By: lcian <17258265+lcian@users.noreply.github.com> --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 94c77c1d31a2..7b5cf7fc3513 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ dependencies = [ "mmh3>=4.0.0", "msgspec>=0.19.0", "msgpack>=1.2.1", - "objectstore-client>=0.2.5", + "objectstore-client>=0.2.10", "orjson>=3.10.10", "packaging>=24.1", "parsimonious>=0.10.0", diff --git a/uv.lock b/uv.lock index 5a4e8b593fc4..2e8d4559411f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.13" resolution-markers = [ "(python_full_version < '3.14' and sys_platform == 'darwin') or (python_full_version < '3.14' and sys_platform == 'linux')", @@ -1423,7 +1423,7 @@ wheels = [ [[package]] name = "objectstore-client" -version = "0.2.5" +version = "0.2.10" source = { registry = "https://pypi.devinfra.sentry.io/simple" } dependencies = [ { name = "filetype" }, @@ -1433,7 +1433,7 @@ dependencies = [ { name = "zstandard" }, ] wheels = [ - { url = "https://pypi.devinfra.sentry.io/wheels/objectstore_client-0.2.5-py3-none-any.whl", hash = "sha256:d080ae54573d7b1eb10a6a282dee03de1efa459b54c3dc99c2569b253d0b6dd3" }, + { url = "https://pypi.devinfra.sentry.io/wheels/objectstore_client-0.2.10-py3-none-any.whl", hash = "sha256:62c953d720c96aa28351f38835d9a9628c9b45920d73abd7e8ff895793305c77" }, ] [[package]] @@ -2412,7 +2412,7 @@ requires-dist = [ { name = "mmh3", specifier = ">=4.0.0" }, { name = "msgpack", specifier = ">=1.2.1" }, { name = "msgspec", specifier = ">=0.19.0" }, - { name = "objectstore-client", specifier = ">=0.2.5" }, + { name = "objectstore-client", specifier = ">=0.2.10" }, { name = "orjson", specifier = ">=3.10.10" }, { name = "packaging", specifier = ">=24.1" }, { name = "parsimonious", specifier = ">=0.10.0" }, From cc6f365e23cc11b6708f1d75fd40e54fdc9a64a1 Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:05:39 +0200 Subject: [PATCH 2/7] ref(preprod): Use resumable puts for snapshot archives --- src/sentry/preprod/snapshots/storage.py | 21 ++- src/sentry/preprod/snapshots/zip_tasks.py | 45 +----- .../sentry/preprod/snapshots/test_storage.py | 9 +- .../preprod/snapshots/test_zip_tasks.py | 128 +++++------------- uv.lock | 2 +- 5 files changed, 51 insertions(+), 154 deletions(-) diff --git a/src/sentry/preprod/snapshots/storage.py b/src/sentry/preprod/snapshots/storage.py index 7b8b3fb02041..1256988442e6 100644 --- a/src/sentry/preprod/snapshots/storage.py +++ b/src/sentry/preprod/snapshots/storage.py @@ -1,10 +1,9 @@ from __future__ import annotations import logging -from typing import IO, Literal +from typing import IO from objectstore_client import Compression, GetResponse, Metadata, RequestError, Session -from objectstore_client.multipart import MultipartUpload from urllib3.exceptions import HTTPError from sentry.models.project import Project @@ -48,15 +47,15 @@ def _record_fallback(self, op: str, found: bool) -> None: }, ) - def put(self, contents: bytes | IO[bytes], *, key: str, content_type: str | None = None) -> str: - return self._primary.put(contents, key=key, content_type=content_type) - - def initiate_multipart_upload( - self, *, key: str, compression: Compression | Literal["none"], content_type: str - ) -> MultipartUpload: - return self._primary.initiate_multipart_upload( - key=key, compression=compression, content_type=content_type - ) + def put( + self, + contents: bytes | IO[bytes], + *, + key: str, + compress: Compression | None = None, + content_type: str | None = None, + ) -> str: + return self._primary.put(contents, key=key, compress=compress, content_type=content_type) def delete(self, key: str) -> None: error: RequestError | HTTPError | None = None diff --git a/src/sentry/preprod/snapshots/zip_tasks.py b/src/sentry/preprod/snapshots/zip_tasks.py index 3a7e6978b50b..2b00ebf3e60d 100644 --- a/src/sentry/preprod/snapshots/zip_tasks.py +++ b/src/sentry/preprod/snapshots/zip_tasks.py @@ -2,14 +2,10 @@ import logging import os -import time from tempfile import NamedTemporaryFile -from typing import IO, Any +from typing import Any import orjson -from objectstore_client import RequestError -from objectstore_client.multipart import CompletePart, MultipartUpload -from urllib3.exceptions import HTTPError from sentry.models.organization import Organization from sentry.preprod.snapshots.manifest import SnapshotManifest @@ -29,9 +25,6 @@ logger = logging.getLogger(__name__) -MULTIPART_PART_SIZE = 32 * 1024 * 1024 -MULTIPART_MAX_RETRIES = 3 - def _snapshot_page_url(organization: Organization, artifact_id: int) -> str: # Link to the snapshots UI rather than the raw download endpoint: the page's @@ -66,17 +59,6 @@ def _send_archive_email( ).send_async([email]) -def _put_part_with_retry(upload: MultipartUpload, chunk: bytes, part_number: int) -> CompletePart: - for attempt in range(MULTIPART_MAX_RETRIES): - try: - return upload.put_part(chunk, part_number=part_number, content_length=len(chunk)) - except (RequestError, HTTPError): - if attempt == MULTIPART_MAX_RETRIES - 1: - raise - time.sleep(2**attempt) - raise AssertionError("unreachable") - - def _archive_available(org_id: int, project_id: int, artifact_id: int) -> bool: try: session = get_snapshot_storage(project_id, org=org_id) @@ -85,28 +67,6 @@ def _archive_available(org_id: int, project_id: int, artifact_id: int) -> bool: return False -def _upload_archive_multipart(session: SnapshotStorage, key: str, tmp: IO[bytes]) -> None: - upload = session.initiate_multipart_upload( - key=key, compression="none", content_type="application/zip" - ) - try: - parts: list[CompletePart] = [] - part_number = 1 - while True: - chunk = tmp.read(MULTIPART_PART_SIZE) - if not chunk: - break - parts.append(_put_part_with_retry(upload, chunk, part_number)) - part_number += 1 - upload.complete(parts) - except Exception: - try: - upload.abort() - except (RequestError, HTTPError): - logger.warning("preprod_snapshot_zip.archive_abort_failed", extra={"key": key}) - raise - - @instrumented_task( name="sentry.preprod.tasks.build_snapshot_images_zip", namespace=preprod_snapshots_tasks, @@ -196,7 +156,8 @@ def build_snapshot_images_zip( "archive_size_bytes": archive_size_bytes, }, ) - _upload_archive_multipart(session, key, tmp) + # Pass the seekable file so the client can resume large uploads. + session.put(tmp, key=key, compress="none", content_type="application/zip") logger.info( "preprod_snapshot_zip.upload_completed", extra={ diff --git a/tests/sentry/preprod/snapshots/test_storage.py b/tests/sentry/preprod/snapshots/test_storage.py index 177d55c7c29f..9b31c1538723 100644 --- a/tests/sentry/preprod/snapshots/test_storage.py +++ b/tests/sentry/preprod/snapshots/test_storage.py @@ -49,11 +49,14 @@ def test_head_uses_fallback_when_primary_missing(sessions) -> None: assert storage.head("k") is fallback.head.return_value -def test_put_only_writes_primary(sessions) -> None: +@pytest.mark.parametrize("compress", [None, "none", "zstd"]) +def test_put_only_writes_primary(sessions, compress) -> None: primary, fallback = sessions storage = SnapshotStorage(primary, fallback) - storage.put(b"data", key="k", content_type="image/png") - primary.put.assert_called_once_with(b"data", key="k", content_type="image/png") + storage.put(b"data", key="k", compress=compress, content_type="image/png") + primary.put.assert_called_once_with( + b"data", key="k", compress=compress, content_type="image/png" + ) fallback.put.assert_not_called() diff --git a/tests/sentry/preprod/snapshots/test_zip_tasks.py b/tests/sentry/preprod/snapshots/test_zip_tasks.py index 29d8ea27f1e0..f5d6271932cb 100644 --- a/tests/sentry/preprod/snapshots/test_zip_tasks.py +++ b/tests/sentry/preprod/snapshots/test_zip_tasks.py @@ -5,18 +5,12 @@ from unittest.mock import MagicMock, patch import orjson -import pytest -from objectstore_client import RequestError -from objectstore_client.multipart import CompletePart +from objectstore_client import Client, RequestError, Usecase from sentry.preprod.models import PreprodArtifact -from sentry.preprod.snapshots import zip_tasks from sentry.preprod.snapshots.models import PreprodSnapshotMetrics -from sentry.preprod.snapshots.zip_tasks import ( - _put_part_with_retry, - _upload_archive_multipart, - build_snapshot_images_zip, -) +from sentry.preprod.snapshots.storage import SnapshotStorage +from sentry.preprod.snapshots.zip_tasks import build_snapshot_images_zip from sentry.testutils.cases import TestCase SESSION_TARGET = "sentry.preprod.snapshots.zip_tasks.get_snapshot_storage" @@ -64,28 +58,45 @@ def _get(key): @patch("sentry.preprod.snapshots.zip_tasks._send_archive_email") @patch(SESSION_TARGET) - def test_build_includes_manifest(self, mock_session, mock_email): + @patch("objectstore_client.client._resumable.upload") + def test_build_uses_resumable_upload_and_includes_manifest( + self, mock_upload, mock_session, mock_email + ): session = self._session() mock_session.return_value = session + primary = Client("http://objectstore.invalid").session( + Usecase("preprod_snapshots", resumable_threshold_bytes=1) + ) + session.put.side_effect = SnapshotStorage(primary, MagicMock()).put + archive_bytes = b"" + + def upload(client_session, body, size, **kwargs): + nonlocal archive_bytes + assert body.seekable() + assert body.tell() == 0 + archive_bytes = body.read() + assert len(archive_bytes) == size + return kwargs["key"] + + mock_upload.side_effect = upload build_snapshot_images_zip( org_id=self.org.id, project_id=self.project.id, artifact_id=self.artifact.id, user_id=self.user.id, ) - session.initiate_multipart_upload.assert_called_once() - kwargs = session.initiate_multipart_upload.call_args.kwargs + session.put.assert_called_once() + kwargs = session.put.call_args.kwargs assert kwargs["key"] == f"snapshot_archives/{self.artifact.id}.zip" - assert kwargs["compression"] == "none" + assert kwargs["compress"] == "none" assert kwargs["content_type"] == "application/zip" - upload = session.initiate_multipart_upload.return_value - assert upload.put_part.call_count >= 1 - archive_bytes = b"".join(call.args[0] for call in upload.put_part.call_args_list) + mock_upload.assert_called_once() + assert mock_upload.call_args.args[1] is session.put.call_args.args[0] + assert mock_upload.call_args.kwargs["compression"] == "none" with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive: assert sorted(archive.namelist()) == ["a.png", "manifest.json"] assert archive.read("manifest.json") == _manifest_bytes() - upload.complete.assert_called_once() mock_email.assert_called_once() assert mock_email.call_args.kwargs["ready"] is True @@ -100,7 +111,7 @@ def test_reuses_existing_archive(self, mock_session, mock_email): artifact_id=self.artifact.id, user_id=self.user.id, ) - session.initiate_multipart_upload.assert_not_called() + session.put.assert_not_called() mock_email.assert_called_once() assert mock_email.call_args.kwargs["ready"] is True @@ -108,7 +119,7 @@ def test_reuses_existing_archive(self, mock_session, mock_email): @patch(SESSION_TARGET) def test_emails_failure_when_upload_raises(self, mock_session, mock_email): session = self._session() - session.initiate_multipart_upload.side_effect = RuntimeError("boom") + session.put.side_effect = RuntimeError("boom") mock_session.return_value = session build_snapshot_images_zip( org_id=self.org.id, @@ -126,7 +137,7 @@ def test_emails_ready_when_failure_superseded_by_existing_archive( self, mock_session, mock_email, mock_available ): session = self._session() - session.initiate_multipart_upload.side_effect = RuntimeError("boom") + session.put.side_effect = RuntimeError("boom") mock_session.return_value = session build_snapshot_images_zip( org_id=self.org.id, @@ -174,80 +185,3 @@ def test_emails_failure_when_objectstore_probe_errors(self, mock_session, mock_e ) mock_email.assert_called_once() assert mock_email.call_args.kwargs["ready"] is False - - -class MultipartUploadHelperTest(TestCase): - @patch("sentry.preprod.snapshots.zip_tasks.time.sleep") - def test_put_part_retries_then_succeeds(self, mock_sleep): - upload = MagicMock() - part = CompletePart(part_number=1, etag="e1") - upload.put_part.side_effect = [ - RequestError("transient", status=500, response=""), - part, - ] - - result = _put_part_with_retry(upload, b"data", 1) - - assert result is part - assert upload.put_part.call_count == 2 - - @patch("sentry.preprod.snapshots.zip_tasks.time.sleep") - def test_put_part_retries_on_transport_error(self, mock_sleep): - from urllib3.exceptions import ProtocolError - - upload = MagicMock() - part = CompletePart(part_number=1, etag="e1") - upload.put_part.side_effect = [ProtocolError("connection aborted"), part] - - result = _put_part_with_retry(upload, b"data", 1) - - assert result is part - assert upload.put_part.call_count == 2 - - @patch("sentry.preprod.snapshots.zip_tasks.time.sleep") - def test_put_part_raises_after_max_retries(self, mock_sleep): - upload = MagicMock() - upload.put_part.side_effect = RequestError("down", status=500, response="") - - with pytest.raises(RequestError): - _put_part_with_retry(upload, b"data", 1) - - assert upload.put_part.call_count == 3 - - def test_upload_archive_multipart_uploads_all_parts(self): - upload = MagicMock() - upload.put_part.side_effect = lambda chunk, *, part_number, content_length: CompletePart( - part_number=part_number, etag=f"e{part_number}" - ) - session = MagicMock() - session.initiate_multipart_upload.return_value = upload - - with patch.object(zip_tasks, "MULTIPART_PART_SIZE", 4): - _upload_archive_multipart(session, "snapshot_archives/1.zip", io.BytesIO(b"abcdefghij")) - - session.initiate_multipart_upload.assert_called_once_with( - key="snapshot_archives/1.zip", - compression="none", - content_type="application/zip", - ) - assert upload.put_part.call_count == 3 - upload.complete.assert_called_once() - parts = upload.complete.call_args.args[0] - assert [p.part_number for p in parts] == [1, 2, 3] - upload.abort.assert_not_called() - - @patch("sentry.preprod.snapshots.zip_tasks.time.sleep") - def test_upload_archive_multipart_aborts_on_failure(self, mock_sleep): - upload = MagicMock() - upload.put_part.side_effect = RequestError("down", status=500, response="") - session = MagicMock() - session.initiate_multipart_upload.return_value = upload - - with patch.object(zip_tasks, "MULTIPART_PART_SIZE", 4): - with pytest.raises(RequestError): - _upload_archive_multipart( - session, "snapshot_archives/1.zip", io.BytesIO(b"abcdefgh") - ) - - upload.abort.assert_called_once() - upload.complete.assert_not_called() diff --git a/uv.lock b/uv.lock index 2e8d4559411f..f4fbea628676 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.13" resolution-markers = [ "(python_full_version < '3.14' and sys_platform == 'darwin') or (python_full_version < '3.14' and sys_platform == 'linux')", From 9d0668acba8c9d829c8985a0eb0a97cb790afb8d Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:26:37 +0200 Subject: [PATCH 3/7] ref(preprod): Use default compression for snapshot archives --- src/sentry/preprod/snapshots/storage.py | 13 +++---------- src/sentry/preprod/snapshots/zip_tasks.py | 2 +- tests/sentry/preprod/snapshots/test_storage.py | 9 +++------ tests/sentry/preprod/snapshots/test_zip_tasks.py | 11 ++++++----- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/src/sentry/preprod/snapshots/storage.py b/src/sentry/preprod/snapshots/storage.py index 1256988442e6..b40f55c86ca5 100644 --- a/src/sentry/preprod/snapshots/storage.py +++ b/src/sentry/preprod/snapshots/storage.py @@ -3,7 +3,7 @@ import logging from typing import IO -from objectstore_client import Compression, GetResponse, Metadata, RequestError, Session +from objectstore_client import GetResponse, Metadata, RequestError, Session from urllib3.exceptions import HTTPError from sentry.models.project import Project @@ -47,15 +47,8 @@ def _record_fallback(self, op: str, found: bool) -> None: }, ) - def put( - self, - contents: bytes | IO[bytes], - *, - key: str, - compress: Compression | None = None, - content_type: str | None = None, - ) -> str: - return self._primary.put(contents, key=key, compress=compress, content_type=content_type) + def put(self, contents: bytes | IO[bytes], *, key: str, content_type: str | None = None) -> str: + return self._primary.put(contents, key=key, content_type=content_type) def delete(self, key: str) -> None: error: RequestError | HTTPError | None = None diff --git a/src/sentry/preprod/snapshots/zip_tasks.py b/src/sentry/preprod/snapshots/zip_tasks.py index 2b00ebf3e60d..7eeacef3b4a1 100644 --- a/src/sentry/preprod/snapshots/zip_tasks.py +++ b/src/sentry/preprod/snapshots/zip_tasks.py @@ -157,7 +157,7 @@ def build_snapshot_images_zip( }, ) # Pass the seekable file so the client can resume large uploads. - session.put(tmp, key=key, compress="none", content_type="application/zip") + session.put(tmp, key=key, content_type="application/zip") logger.info( "preprod_snapshot_zip.upload_completed", extra={ diff --git a/tests/sentry/preprod/snapshots/test_storage.py b/tests/sentry/preprod/snapshots/test_storage.py index 9b31c1538723..177d55c7c29f 100644 --- a/tests/sentry/preprod/snapshots/test_storage.py +++ b/tests/sentry/preprod/snapshots/test_storage.py @@ -49,14 +49,11 @@ def test_head_uses_fallback_when_primary_missing(sessions) -> None: assert storage.head("k") is fallback.head.return_value -@pytest.mark.parametrize("compress", [None, "none", "zstd"]) -def test_put_only_writes_primary(sessions, compress) -> None: +def test_put_only_writes_primary(sessions) -> None: primary, fallback = sessions storage = SnapshotStorage(primary, fallback) - storage.put(b"data", key="k", compress=compress, content_type="image/png") - primary.put.assert_called_once_with( - b"data", key="k", compress=compress, content_type="image/png" - ) + storage.put(b"data", key="k", content_type="image/png") + primary.put.assert_called_once_with(b"data", key="k", content_type="image/png") fallback.put.assert_not_called() diff --git a/tests/sentry/preprod/snapshots/test_zip_tasks.py b/tests/sentry/preprod/snapshots/test_zip_tasks.py index f5d6271932cb..250a3dca5d6f 100644 --- a/tests/sentry/preprod/snapshots/test_zip_tasks.py +++ b/tests/sentry/preprod/snapshots/test_zip_tasks.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patch import orjson +import zstandard from objectstore_client import Client, RequestError, Usecase from sentry.preprod.models import PreprodArtifact @@ -74,8 +75,10 @@ def upload(client_session, body, size, **kwargs): nonlocal archive_bytes assert body.seekable() assert body.tell() == 0 - archive_bytes = body.read() - assert len(archive_bytes) == size + compressed_bytes = body.read() + assert len(compressed_bytes) == size + with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(compressed_bytes)) as reader: + archive_bytes = reader.read() return kwargs["key"] mock_upload.side_effect = upload @@ -88,12 +91,10 @@ def upload(client_session, body, size, **kwargs): session.put.assert_called_once() kwargs = session.put.call_args.kwargs assert kwargs["key"] == f"snapshot_archives/{self.artifact.id}.zip" - assert kwargs["compress"] == "none" assert kwargs["content_type"] == "application/zip" mock_upload.assert_called_once() - assert mock_upload.call_args.args[1] is session.put.call_args.args[0] - assert mock_upload.call_args.kwargs["compression"] == "none" + assert mock_upload.call_args.kwargs["compression"] == "zstd" with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive: assert sorted(archive.namelist()) == ["a.png", "manifest.json"] assert archive.read("manifest.json") == _manifest_bytes() From 012c030fa9271d7c36f998d11b90f9a5e6910832 Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:36:44 +0200 Subject: [PATCH 4/7] improve --- src/sentry/preprod/snapshots/zip_tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sentry/preprod/snapshots/zip_tasks.py b/src/sentry/preprod/snapshots/zip_tasks.py index 7eeacef3b4a1..3c77339b5041 100644 --- a/src/sentry/preprod/snapshots/zip_tasks.py +++ b/src/sentry/preprod/snapshots/zip_tasks.py @@ -156,7 +156,6 @@ def build_snapshot_images_zip( "archive_size_bytes": archive_size_bytes, }, ) - # Pass the seekable file so the client can resume large uploads. session.put(tmp, key=key, content_type="application/zip") logger.info( "preprod_snapshot_zip.upload_completed", From 8bbc763e53d3e6f0d8ba416e4b643843f80dd717 Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:38:20 +0200 Subject: [PATCH 5/7] test(preprod): Keep archive coverage focused on ZIP contents --- .../preprod/snapshots/test_zip_tasks.py | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/tests/sentry/preprod/snapshots/test_zip_tasks.py b/tests/sentry/preprod/snapshots/test_zip_tasks.py index 250a3dca5d6f..dd6925bc281b 100644 --- a/tests/sentry/preprod/snapshots/test_zip_tasks.py +++ b/tests/sentry/preprod/snapshots/test_zip_tasks.py @@ -5,12 +5,10 @@ from unittest.mock import MagicMock, patch import orjson -import zstandard -from objectstore_client import Client, RequestError, Usecase +from objectstore_client import RequestError from sentry.preprod.models import PreprodArtifact from sentry.preprod.snapshots.models import PreprodSnapshotMetrics -from sentry.preprod.snapshots.storage import SnapshotStorage from sentry.preprod.snapshots.zip_tasks import build_snapshot_images_zip from sentry.testutils.cases import TestCase @@ -59,29 +57,17 @@ def _get(key): @patch("sentry.preprod.snapshots.zip_tasks._send_archive_email") @patch(SESSION_TARGET) - @patch("objectstore_client.client._resumable.upload") - def test_build_uses_resumable_upload_and_includes_manifest( - self, mock_upload, mock_session, mock_email - ): + def test_build_includes_manifest(self, mock_session, mock_email): session = self._session() mock_session.return_value = session - primary = Client("http://objectstore.invalid").session( - Usecase("preprod_snapshots", resumable_threshold_bytes=1) - ) - session.put.side_effect = SnapshotStorage(primary, MagicMock()).put archive_bytes = b"" - def upload(client_session, body, size, **kwargs): + def upload(contents, **kwargs): nonlocal archive_bytes - assert body.seekable() - assert body.tell() == 0 - compressed_bytes = body.read() - assert len(compressed_bytes) == size - with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(compressed_bytes)) as reader: - archive_bytes = reader.read() + archive_bytes = contents.read() return kwargs["key"] - mock_upload.side_effect = upload + session.put.side_effect = upload build_snapshot_images_zip( org_id=self.org.id, project_id=self.project.id, @@ -93,8 +79,6 @@ def upload(client_session, body, size, **kwargs): assert kwargs["key"] == f"snapshot_archives/{self.artifact.id}.zip" assert kwargs["content_type"] == "application/zip" - mock_upload.assert_called_once() - assert mock_upload.call_args.kwargs["compression"] == "zstd" with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive: assert sorted(archive.namelist()) == ["a.png", "manifest.json"] assert archive.read("manifest.json") == _manifest_bytes() From aedc3c85a8257e65b8141351ecd8fa27aded1cdd Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:19:04 +0200 Subject: [PATCH 6/7] fix(preprod): Keep snapshot ZIP uploads uncompressed and resumable --- src/sentry/preprod/snapshots/storage.py | 20 ++++++++++++++++--- src/sentry/preprod/snapshots/zip_tasks.py | 8 +++++++- .../sentry/preprod/snapshots/test_storage.py | 8 ++++++-- .../preprod/snapshots/test_zip_tasks.py | 2 ++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/sentry/preprod/snapshots/storage.py b/src/sentry/preprod/snapshots/storage.py index b40f55c86ca5..99c5afe64aa4 100644 --- a/src/sentry/preprod/snapshots/storage.py +++ b/src/sentry/preprod/snapshots/storage.py @@ -3,7 +3,7 @@ import logging from typing import IO -from objectstore_client import GetResponse, Metadata, RequestError, Session +from objectstore_client import Compression, GetResponse, Metadata, RequestError, Session from urllib3.exceptions import HTTPError from sentry.models.project import Project @@ -47,8 +47,22 @@ def _record_fallback(self, op: str, found: bool) -> None: }, ) - def put(self, contents: bytes | IO[bytes], *, key: str, content_type: str | None = None) -> str: - return self._primary.put(contents, key=key, content_type=content_type) + def put( + self, + contents: bytes | IO[bytes], + *, + key: str, + content_type: str | None = None, + compress: Compression | None = None, + resumable_threshold_bytes: int | None = None, + ) -> str: + return self._primary.put( + contents, + key=key, + content_type=content_type, + compress=compress, + resumable_threshold_bytes=resumable_threshold_bytes, + ) def delete(self, key: str) -> None: error: RequestError | HTTPError | None = None diff --git a/src/sentry/preprod/snapshots/zip_tasks.py b/src/sentry/preprod/snapshots/zip_tasks.py index 3c77339b5041..2f4b4a99a413 100644 --- a/src/sentry/preprod/snapshots/zip_tasks.py +++ b/src/sentry/preprod/snapshots/zip_tasks.py @@ -156,7 +156,13 @@ def build_snapshot_images_zip( "archive_size_bytes": archive_size_bytes, }, ) - session.put(tmp, key=key, content_type="application/zip") + session.put( + tmp, + key=key, + content_type="application/zip", + compress="none", + resumable_threshold_bytes=0, + ) logger.info( "preprod_snapshot_zip.upload_completed", extra={ diff --git a/tests/sentry/preprod/snapshots/test_storage.py b/tests/sentry/preprod/snapshots/test_storage.py index 177d55c7c29f..a24ed5c1577e 100644 --- a/tests/sentry/preprod/snapshots/test_storage.py +++ b/tests/sentry/preprod/snapshots/test_storage.py @@ -52,8 +52,12 @@ def test_head_uses_fallback_when_primary_missing(sessions) -> None: def test_put_only_writes_primary(sessions) -> None: primary, fallback = sessions storage = SnapshotStorage(primary, fallback) - storage.put(b"data", key="k", content_type="image/png") - primary.put.assert_called_once_with(b"data", key="k", content_type="image/png") + storage.put( + b"data", key="k", content_type="image/png", compress="none", resumable_threshold_bytes=0 + ) + primary.put.assert_called_once_with( + b"data", key="k", content_type="image/png", compress="none", resumable_threshold_bytes=0 + ) fallback.put.assert_not_called() diff --git a/tests/sentry/preprod/snapshots/test_zip_tasks.py b/tests/sentry/preprod/snapshots/test_zip_tasks.py index dd6925bc281b..6a78c80a221f 100644 --- a/tests/sentry/preprod/snapshots/test_zip_tasks.py +++ b/tests/sentry/preprod/snapshots/test_zip_tasks.py @@ -78,6 +78,8 @@ def upload(contents, **kwargs): kwargs = session.put.call_args.kwargs assert kwargs["key"] == f"snapshot_archives/{self.artifact.id}.zip" assert kwargs["content_type"] == "application/zip" + assert kwargs["compress"] == "none" + assert kwargs["resumable_threshold_bytes"] == 0 with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive: assert sorted(archive.namelist()) == ["a.png", "manifest.json"] From 1e1f7710ace88b2b8babbe4d5950027a650eff52 Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Wed, 21 Oct 2026 13:24:10 +0200 Subject: [PATCH 7/7] add comments --- src/sentry/preprod/snapshots/zip_tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sentry/preprod/snapshots/zip_tasks.py b/src/sentry/preprod/snapshots/zip_tasks.py index 2f4b4a99a413..1638ceec3b8d 100644 --- a/src/sentry/preprod/snapshots/zip_tasks.py +++ b/src/sentry/preprod/snapshots/zip_tasks.py @@ -160,7 +160,11 @@ def build_snapshot_images_zip( tmp, key=key, content_type="application/zip", + # A `zip` is already compressed, so it would likely be not beneficial to compress it again. compress="none", + # Force this `put` to use a resumable upload, overriding the default threshold. + # This already incorporates retries with exponential backoffs and allows the client to retry + # from the persisted offset when the upload fails midway through. resumable_threshold_bytes=0, ) logger.info(