fix: persist multipart uploads as durable jobs
This commit is contained in:
parent
aa9638f6b3
commit
9dd5f386ce
6 changed files with 54 additions and 2 deletions
|
|
@ -147,6 +147,7 @@ class UploadItem(Base):
|
|||
width: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
height: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
r2_key: Mapped[str] = mapped_column(String(1024))
|
||||
multipart_upload_id: Mapped[str | None] = mapped_column(String(512), nullable=True, index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
|
|
|||
|
|
@ -126,6 +126,12 @@ def abort_multipart(key: str, upload_id: str) -> None:
|
|||
pass
|
||||
|
||||
|
||||
def list_multipart_parts(key: str, upload_id: str) -> list[dict]:
|
||||
s = get_settings()
|
||||
response = _client().list_parts(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
|
||||
return response.get("Parts", [])
|
||||
|
||||
|
||||
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
|
||||
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
|
||||
s = get_settings()
|
||||
|
|
|
|||
|
|
@ -71,6 +71,15 @@ async def init_db():
|
|||
from src.core.security import hash_password
|
||||
async with get_engine().begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
if conn.dialect.name == "postgresql":
|
||||
await conn.execute(text(
|
||||
"ALTER TABLE upload_items ADD COLUMN IF NOT EXISTS "
|
||||
"multipart_upload_id VARCHAR(512)"
|
||||
))
|
||||
await conn.execute(text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_upload_items_multipart_upload_id "
|
||||
"ON upload_items (multipart_upload_id)"
|
||||
))
|
||||
factory = get_session_factory()
|
||||
async with factory() as s:
|
||||
admin = (await s.execute(select(User).where(User.email == "admin@danagble.local"))).scalar_one_or_none()
|
||||
|
|
|
|||
|
|
@ -36,6 +36,11 @@ async def create(project_id: str, body: CreateUploadJobIn,
|
|||
return await svc.create_job(project_id, body.kind, [f.model_dump() for f in body.files], db)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_active(project_id: str, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.list_active(project_id, db)
|
||||
|
||||
|
||||
@router.get("/{job_id}")
|
||||
async def get(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
||||
data = await svc.get_job_detail(job_id, db)
|
||||
|
|
|
|||
|
|
@ -9,9 +9,10 @@ from __future__ import annotations
|
|||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Project
|
||||
from models import Project, UploadItem, UploadJob
|
||||
from shared import r2
|
||||
from src.core.db import get_session
|
||||
|
||||
|
|
@ -64,7 +65,17 @@ async def start_multipart(project_id: str, body: MultipartStart,
|
|||
upload_id = r2.create_multipart(key, body.contentType)
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
|
||||
job = UploadJob(project_id=project_id, kind="zip", total_files=1)
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
item = UploadItem(project_id=project_id, job_id=job.id,
|
||||
file_name=body.fileName,
|
||||
content_type=body.contentType,
|
||||
r2_key=key, multipart_upload_id=upload_id)
|
||||
db.add(item)
|
||||
await db.commit()
|
||||
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE,
|
||||
"jobId": job.id, "itemId": item.id}
|
||||
|
||||
|
||||
@router.post("/multipart/parts")
|
||||
|
|
@ -91,6 +102,18 @@ async def complete_multipart(project_id: str, body: MultipartComplete,
|
|||
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
|
||||
if not r2.head_exists(body.key):
|
||||
raise HTTPException(400, "object not found after complete")
|
||||
item = (await db.execute(select(UploadItem).where(
|
||||
UploadItem.project_id == project_id,
|
||||
UploadItem.r2_key == body.key,
|
||||
UploadItem.multipart_upload_id == body.uploadId
|
||||
))).scalar_one_or_none()
|
||||
if item is not None:
|
||||
item.status = "uploaded"
|
||||
job = await db.get(UploadJob, item.job_id)
|
||||
if job is not None:
|
||||
job.status = "uploaded"
|
||||
job.uploaded_files = 1
|
||||
await db.commit()
|
||||
return {"key": body.key, "etag": etag}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -59,6 +59,14 @@ async def get_job(job_id: str, db: AsyncSession) -> dict:
|
|||
return _job_dict(job)
|
||||
|
||||
|
||||
async def list_active(project_id: str, db: AsyncSession) -> list[dict]:
|
||||
rows = (await db.execute(select(UploadJob).where(
|
||||
UploadJob.project_id == project_id,
|
||||
UploadJob.status.in_(["uploading", "uploaded", "processing"]),
|
||||
).order_by(UploadJob.created_at.desc()))).scalars().all()
|
||||
return [_job_dict(job) for job in rows]
|
||||
|
||||
|
||||
async def get_job_detail(job_id: str, db: AsyncSession) -> dict:
|
||||
job = await db.get(UploadJob, job_id)
|
||||
if job is None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue