Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 0 additions & 3 deletions clients/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,6 @@
### Documentation 📚

- Fix and expand documentation across the workspace by @jan-auer in [#521](https://github.com/getsentry/objectstore/pull/521)
- Fix multipart endpoint paths and client examples by @jan-auer in [#505](https://github.com/getsentry/objectstore/pull/505)

### Internal Changes 🔧

Expand All @@ -167,8 +166,6 @@

### New Features ✨

- (py-client) Add Multipart Upload API by @lcian in [#476](https://github.com/getsentry/objectstore/pull/476)
- (rust-client) Add Multipart Upload API by @lcian in [#468](https://github.com/getsentry/objectstore/pull/468)

## 0.1.9

Expand Down
60 changes: 0 additions & 60 deletions clients/python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,66 +116,6 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
session.put(b"payload", metadata={"source": "upload-service"})
```

### Multipart Upload API

For large objects, use multipart uploads to upload parts independently and then
assemble them into a final object.

**Important:** unlike single-object uploads, multipart uploads do **not** auto-compress.
`compression` on `initiate_multipart_upload` behaves like `precompressed` above: it only
records the algorithm, and the caller must pre-compress each part accordingly.

```python
from concurrent.futures import ThreadPoolExecutor

import zstandard

from objectstore_client.multipart import MultipartCompleteError

upload = session.initiate_multipart_upload(
key="my-large-object",
compression="zstd",
metadata={"source": "upload-service"},
)

compressor = zstandard.ZstdCompressor()
chunks = [b"part1", b"part2", b"part3", b"part4"]

def upload_part(part_number: int, data: bytes):
compressed = compressor.compress(data)
return upload.put_part(
compressed, part_number=part_number, content_length=len(compressed)
)

with ThreadPoolExecutor(max_workers=4) as executor:
futures = [
executor.submit(upload_part, i + 1, chunk)
for i, chunk in enumerate(chunks)
]
parts = [f.result() for f in futures]

try:
key = upload.complete(parts)
except MultipartCompleteError:
upload.abort()
raise
```

To resume an in-progress multipart upload after a process restart, persist the
`key` and `upload_id`, then reconstruct the upload handle later:

```python
saved_key = upload.key
saved_upload_id = upload.upload_id

resumed = session.resume_multipart_upload(saved_key, saved_upload_id)
existing_parts = resumed.list_parts()

# Upload missing parts...

key = resumed.complete(new_parts + existing_parts)
```

### Authentication

If your Objectstore instance enforces authorization, you must configure authentication
Expand Down
8 changes: 0 additions & 8 deletions clients/python/docs/objectstore_client.rst
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,6 @@ objectstore\_client.metrics module
:show-inheritance:
:undoc-members:

objectstore\_client.multipart module
------------------------------------

.. automodule:: objectstore_client.multipart
:members:
:show-inheritance:
:undoc-members:

objectstore\_client.presign module
----------------------------------

Expand Down
92 changes: 1 addition & 91 deletions clients/python/src/objectstore_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
NoOpMetricsBackend,
measure_storage_operation,
)
from objectstore_client.multipart import MultipartUpload
from objectstore_client.scope import Scope
from objectstore_client.tracing import storage_span

Expand Down Expand Up @@ -94,7 +93,7 @@ def __init__(
"usecase name must contain only ASCII letters, digits, '_' or '-'"
)
self._name = name

if resumable_threshold_bytes != "none" and (
type(resumable_threshold_bytes) is not int or resumable_threshold_bytes < 0
):
Expand Down Expand Up @@ -400,25 +399,6 @@ def _make_url(self, key: str | None, full: bool = False) -> str:
return f"{self._base_url()}{path}"
return path

def _make_multipart_url(
self,
action: str | None,
key: str | None,
query: str | None = None,
) -> str:
if action == "parts":
resource = "objects:multipart:parts"
elif action == "complete":
resource = "objects:multipart:complete"
else:
resource = "objects:multipart"

relative_path = f"/v1/{resource}/{self._usecase.name}/{self._scope}/{key or ''}"
path = utils.encode_path(self._base_path.rstrip("/") + relative_path)
if query:
return f"{path}?{query}"
return path

def put(
self,
contents: bytes | IO[bytes],
Expand Down Expand Up @@ -1003,73 +983,3 @@ def _create_upload(
def _resume_upload(self, key: str, token: str) -> _resumable.ResumableUpload:
"""Reconstruct a resumable upload handle, without contacting the server."""
return _resumable.ResumableUpload(self, key, token)

def initiate_multipart_upload(
self,
*,
key: str | None = None,
compression: Compression | Literal["none"] | None = None,
content_type: str | None = None,
metadata: dict[str, str] | None = None,
expiration_policy: ExpirationPolicy | None = None,
origin: str | None = None,
filename: str | None = None,
) -> MultipartUpload:
"""
Initiates a multipart upload.

Returns a :class:`~objectstore_client.multipart.MultipartUpload` handle
that can be used to upload parts, list parts, complete, or abort.

**Important:** unlike :meth:`put`, the ``compression`` parameter only
records the compression algorithm in the object's metadata.
The caller is responsible for compressing each part in accordance with the
chosen algorithm before passing it to
:meth:`~objectstore_client.multipart.MultipartUpload.upload_part`.
"""
if compression and compression not in ("none", "zstd"):
raise ValueError(f"Invalid compression: {compression}")

headers = self._metadata_headers(
compression=compression or self._usecase._compression,
content_type=content_type,
metadata=metadata,
expiration_policy=expiration_policy,
origin=origin,
filename=filename,
)

if key == "":
key = None

with (
storage_span(
"multipart.initiate", self._usecase, self._scope, key=key
) as span,
measure_storage_operation(
self._metrics_backend, "multipart.initiate", self._usecase.name
),
):
response = self._pool.request(
"POST" if not key else "PUT",
self._make_multipart_url(None, key),
headers=headers,
preload_content=True,
decode_content=True,
)
raise_for_status(response)
res = response.json()
# Set after the response, since the key may be server-generated.
span.set_attribute("objectstore.key", res["key"])
span.set_attribute("objectstore.upload_id", res["upload_id"])
return MultipartUpload(self, res["key"], res["upload_id"])

def resume_multipart_upload(self, key: str, upload_id: str) -> MultipartUpload:
"""
Reconstructs a multipart upload handle.

This does not make any network calls.
Use it to resume an upload after a process restart or to
continue an upload started elsewhere.
"""
return MultipartUpload(self, key, upload_id)
Loading
Loading