feat: add resumable dataset uploads

This commit is contained in:
hailt 2026-11-12 17:31:42 +07:00
parent f8ed08b854
commit 8d40cff227
23 changed files with 913 additions and 21 deletions

View file

@ -64,6 +64,67 @@ def get_bytes(key: str) -> bytes:
return obj["Body"].read()
def bucket_usage(prefix: str = "") -> dict:
"""Sum bytes + object count via paginated LIST (no download)."""
s = get_settings()
total_bytes, total_objects = 0, 0
paginator = _client().get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=s.r2_bucket, Prefix=prefix):
for obj in page.get("Contents", []):
total_bytes += obj.get("Size", 0)
total_objects += 1
return {"bytes": total_bytes, "objects": total_objects}
def create_multipart(key: str, content_type: str = "application/octet-stream") -> str:
"""Start a multipart upload; returns upload_id. Parts upload in parallel,
each retriable; the upload survives network drops via the stored id."""
s = get_settings()
res = _client().create_multipart_upload(
Bucket=s.r2_bucket, Key=key, ContentType=content_type)
return res["UploadId"]
def presign_part(key: str, upload_id: str, part_number: int,
expires_in: int = 3600) -> str:
s = get_settings()
return _client().generate_presigned_url(
"upload_part",
Params={"Bucket": s.r2_bucket, "Key": key,
"UploadId": upload_id, "PartNumber": part_number},
ExpiresIn=expires_in)
def complete_multipart(key: str, upload_id: str, parts: list[dict]) -> str:
"""parts: [{ETag, PartNumber}]. Returns final ETag."""
s = get_settings()
res = _client().complete_multipart_upload(
Bucket=s.r2_bucket, Key=key, UploadId=upload_id,
MultipartUpload={"Parts": [{"ETag": p["ETag"], "PartNumber": p["PartNumber"]}
for p in sorted(parts, key=lambda x: x["PartNumber"])]})
return res.get("ETag", "")
def abort_multipart(key: str, upload_id: str) -> None:
s = get_settings()
try:
_client().abort_multipart_upload(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
except Exception:
pass
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
s = get_settings()
body = _client().get_object(Bucket=s.r2_bucket, Key=key)["Body"]
total = 0
with open(dest_path, "wb") as f:
for chunk in body.iter_chunks(chunk_size=chunk_bytes):
f.write(chunk)
total += len(chunk)
return total
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
cache_control: str | None = None) -> None:
s = get_settings()