feat: add resumable dataset uploads
This commit is contained in:
parent
f8ed08b854
commit
8d40cff227
23 changed files with 913 additions and 21 deletions
61
shared/r2.py
61
shared/r2.py
|
|
@ -64,6 +64,67 @@ def get_bytes(key: str) -> bytes:
|
|||
return obj["Body"].read()
|
||||
|
||||
|
||||
def bucket_usage(prefix: str = "") -> dict:
|
||||
"""Sum bytes + object count via paginated LIST (no download)."""
|
||||
s = get_settings()
|
||||
total_bytes, total_objects = 0, 0
|
||||
paginator = _client().get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=s.r2_bucket, Prefix=prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
total_bytes += obj.get("Size", 0)
|
||||
total_objects += 1
|
||||
return {"bytes": total_bytes, "objects": total_objects}
|
||||
|
||||
|
||||
def create_multipart(key: str, content_type: str = "application/octet-stream") -> str:
|
||||
"""Start a multipart upload; returns upload_id. Parts upload in parallel,
|
||||
each retriable; the upload survives network drops via the stored id."""
|
||||
s = get_settings()
|
||||
res = _client().create_multipart_upload(
|
||||
Bucket=s.r2_bucket, Key=key, ContentType=content_type)
|
||||
return res["UploadId"]
|
||||
|
||||
|
||||
def presign_part(key: str, upload_id: str, part_number: int,
|
||||
expires_in: int = 3600) -> str:
|
||||
s = get_settings()
|
||||
return _client().generate_presigned_url(
|
||||
"upload_part",
|
||||
Params={"Bucket": s.r2_bucket, "Key": key,
|
||||
"UploadId": upload_id, "PartNumber": part_number},
|
||||
ExpiresIn=expires_in)
|
||||
|
||||
|
||||
def complete_multipart(key: str, upload_id: str, parts: list[dict]) -> str:
|
||||
"""parts: [{ETag, PartNumber}]. Returns final ETag."""
|
||||
s = get_settings()
|
||||
res = _client().complete_multipart_upload(
|
||||
Bucket=s.r2_bucket, Key=key, UploadId=upload_id,
|
||||
MultipartUpload={"Parts": [{"ETag": p["ETag"], "PartNumber": p["PartNumber"]}
|
||||
for p in sorted(parts, key=lambda x: x["PartNumber"])]})
|
||||
return res.get("ETag", "")
|
||||
|
||||
|
||||
def abort_multipart(key: str, upload_id: str) -> None:
|
||||
s = get_settings()
|
||||
try:
|
||||
_client().abort_multipart_upload(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
|
||||
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
|
||||
s = get_settings()
|
||||
body = _client().get_object(Bucket=s.r2_bucket, Key=key)["Body"]
|
||||
total = 0
|
||||
with open(dest_path, "wb") as f:
|
||||
for chunk in body.iter_chunks(chunk_size=chunk_bytes):
|
||||
f.write(chunk)
|
||||
total += len(chunk)
|
||||
return total
|
||||
|
||||
|
||||
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
|
||||
cache_control: str | None = None) -> None:
|
||||
s = get_settings()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue