fix: persist multipart uploads as durable jobs

This commit is contained in:
hailt 2026-11-12 23:09:54 +07:00
parent aa9638f6b3
commit 9dd5f386ce
6 changed files with 54 additions and 2 deletions

View file

@ -147,6 +147,7 @@ class UploadItem(Base):
width: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
height: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
r2_key: Mapped[str] = mapped_column(String(1024))
multipart_upload_id: Mapped[str | None] = mapped_column(String(512), nullable=True, index=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)

View file

@ -126,6 +126,12 @@ def abort_multipart(key: str, upload_id: str) -> None:
pass
def list_multipart_parts(key: str, upload_id: str) -> list[dict]:
s = get_settings()
response = _client().list_parts(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
return response.get("Parts", [])
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
s = get_settings()

View file

@ -71,6 +71,15 @@ async def init_db():
from src.core.security import hash_password
async with get_engine().begin() as conn:
await conn.run_sync(Base.metadata.create_all)
if conn.dialect.name == "postgresql":
await conn.execute(text(
"ALTER TABLE upload_items ADD COLUMN IF NOT EXISTS "
"multipart_upload_id VARCHAR(512)"
))
await conn.execute(text(
"CREATE INDEX IF NOT EXISTS ix_upload_items_multipart_upload_id "
"ON upload_items (multipart_upload_id)"
))
factory = get_session_factory()
async with factory() as s:
admin = (await s.execute(select(User).where(User.email == "admin@danagble.local"))).scalar_one_or_none()

View file

@ -36,6 +36,11 @@ async def create(project_id: str, body: CreateUploadJobIn,
return await svc.create_job(project_id, body.kind, [f.model_dump() for f in body.files], db)
@router.get("")
async def list_active(project_id: str, db: AsyncSession = Depends(get_session)):
return await svc.list_active(project_id, db)
@router.get("/{job_id}")
async def get(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
data = await svc.get_job_detail(job_id, db)

View file

@ -9,9 +9,10 @@ from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models import Project
from models import Project, UploadItem, UploadJob
from shared import r2
from src.core.db import get_session
@ -64,7 +65,17 @@ async def start_multipart(project_id: str, body: MultipartStart,
upload_id = r2.create_multipart(key, body.contentType)
except Exception as e:
raise HTTPException(503, f"storage unreachable: {e}")
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
job = UploadJob(project_id=project_id, kind="zip", total_files=1)
db.add(job)
await db.flush()
item = UploadItem(project_id=project_id, job_id=job.id,
file_name=body.fileName,
content_type=body.contentType,
r2_key=key, multipart_upload_id=upload_id)
db.add(item)
await db.commit()
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE,
"jobId": job.id, "itemId": item.id}
@router.post("/multipart/parts")
@ -91,6 +102,18 @@ async def complete_multipart(project_id: str, body: MultipartComplete,
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
if not r2.head_exists(body.key):
raise HTTPException(400, "object not found after complete")
item = (await db.execute(select(UploadItem).where(
UploadItem.project_id == project_id,
UploadItem.r2_key == body.key,
UploadItem.multipart_upload_id == body.uploadId
))).scalar_one_or_none()
if item is not None:
item.status = "uploaded"
job = await db.get(UploadJob, item.job_id)
if job is not None:
job.status = "uploaded"
job.uploaded_files = 1
await db.commit()
return {"key": body.key, "etag": etag}

View file

@ -59,6 +59,14 @@ async def get_job(job_id: str, db: AsyncSession) -> dict:
return _job_dict(job)
async def list_active(project_id: str, db: AsyncSession) -> list[dict]:
rows = (await db.execute(select(UploadJob).where(
UploadJob.project_id == project_id,
UploadJob.status.in_(["uploading", "uploaded", "processing"]),
).order_by(UploadJob.created_at.desc()))).scalars().all()
return [_job_dict(job) for job in rows]
async def get_job_detail(job_id: str, db: AsyncSession) -> dict:
job = await db.get(UploadJob, job_id)
if job is None: