backend/shared/r2.py

154 lines
5.3 KiB
Python

"""Thin boto3 wrapper for R2 (S3-compatible). No image bytes pass through the API."""
from __future__ import annotations
from functools import lru_cache
from urllib.parse import urlsplit, urlunsplit
from src.core.config import get_settings
@lru_cache
def _client():
try:
import boto3
from botocore.config import Config
except ImportError as e:
raise RuntimeError("boto3 not installed: pip install -r requirements.txt") from e
s = get_settings()
if not s.r2_configured:
raise RuntimeError("R2 not configured: set R2_ENDPOINT/R2_ACCESS_KEY_ID/R2_SECRET in .env")
return boto3.client(
"s3",
endpoint_url=s.r2_endpoint,
aws_access_key_id=s.r2_key_id,
aws_secret_access_key=s.r2_secret,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
region_name="auto",
)
def _publicize(url: str) -> str:
"""Expose an internal signed URL through the HTTPS storage proxy."""
public_base = get_settings().r2_public_base.rstrip("/")
if not public_base:
return url
public = urlsplit(public_base)
signed = urlsplit(url)
return urlunsplit((public.scheme, public.netloc,
f"{public.path}{signed.path}", signed.query, signed.fragment))
def presign_put(key: str, content_type: str = "application/octet-stream") -> str:
"""Direct-upload URL: browser/app PUTs bytes straight to R2 (saves Class-A + bandwidth)."""
s = get_settings()
url = _client().generate_presigned_url(
"put_object",
Params={"Bucket": s.r2_bucket, "Key": key, "ContentType": content_type},
ExpiresIn=s.presign_put_ttl,
)
return _publicize(url)
def presign_get(key: str) -> str:
"""Short-lived read URL for preview/download. Thumbnails preferred over full images."""
s = get_settings()
url = _client().generate_presigned_url(
"get_object",
Params={"Bucket": s.r2_bucket, "Key": key},
ExpiresIn=s.presign_get_ttl,
)
return _publicize(url)
def head_exists(key: str) -> bool:
s = get_settings()
try:
_client().head_object(Bucket=s.r2_bucket, Key=key)
return True
except Exception:
return False
def get_bytes(key: str) -> bytes:
"""Download an object into memory (for workers/thumbnails, not hot paths)."""
s = get_settings()
obj = _client().get_object(Bucket=s.r2_bucket, Key=key)
return obj["Body"].read()
def bucket_usage(prefix: str = "") -> dict:
"""Sum bytes + object count via paginated LIST (no download)."""
s = get_settings()
total_bytes, total_objects = 0, 0
paginator = _client().get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=s.r2_bucket, Prefix=prefix):
for obj in page.get("Contents", []):
total_bytes += obj.get("Size", 0)
total_objects += 1
return {"bytes": total_bytes, "objects": total_objects}
def create_multipart(key: str, content_type: str = "application/octet-stream") -> str:
"""Start a multipart upload; returns upload_id. Parts upload in parallel,
each retriable; the upload survives network drops via the stored id."""
s = get_settings()
res = _client().create_multipart_upload(
Bucket=s.r2_bucket, Key=key, ContentType=content_type)
return res["UploadId"]
def presign_part(key: str, upload_id: str, part_number: int,
expires_in: int = 3600) -> str:
s = get_settings()
url = _client().generate_presigned_url(
"upload_part",
Params={"Bucket": s.r2_bucket, "Key": key,
"UploadId": upload_id, "PartNumber": part_number},
ExpiresIn=expires_in)
return _publicize(url)
def complete_multipart(key: str, upload_id: str, parts: list[dict]) -> str:
"""parts: [{ETag, PartNumber}]. Returns final ETag."""
s = get_settings()
res = _client().complete_multipart_upload(
Bucket=s.r2_bucket, Key=key, UploadId=upload_id,
MultipartUpload={"Parts": [{"ETag": p["ETag"], "PartNumber": p["PartNumber"]}
for p in sorted(parts, key=lambda x: x["PartNumber"])]})
return res.get("ETag", "")
def abort_multipart(key: str, upload_id: str) -> None:
s = get_settings()
try:
_client().abort_multipart_upload(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
except Exception:
pass
def list_multipart_parts(key: str, upload_id: str) -> list[dict]:
s = get_settings()
response = _client().list_parts(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
return response.get("Parts", [])
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
s = get_settings()
body = _client().get_object(Bucket=s.r2_bucket, Key=key)["Body"]
total = 0
with open(dest_path, "wb") as f:
for chunk in body.iter_chunks(chunk_size=chunk_bytes):
f.write(chunk)
total += len(chunk)
return total
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
cache_control: str | None = None) -> None:
s = get_settings()
params: dict = {"Bucket": s.r2_bucket, "Key": key, "Body": data,
"ContentType": content_type}
if cache_control:
params["CacheControl"] = cache_control
_client().put_object(**params)