fix: persist multipart uploads as durable jobs
This commit is contained in:
parent
aa9638f6b3
commit
9dd5f386ce
6 changed files with 54 additions and 2 deletions
|
|
@ -147,6 +147,7 @@ class UploadItem(Base):
|
||||||
width: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
width: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
height: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
height: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
r2_key: Mapped[str] = mapped_column(String(1024))
|
r2_key: Mapped[str] = mapped_column(String(1024))
|
||||||
|
multipart_upload_id: Mapped[str | None] = mapped_column(String(512), nullable=True, index=True)
|
||||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||||
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|
|
||||||
|
|
@ -126,6 +126,12 @@ def abort_multipart(key: str, upload_id: str) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def list_multipart_parts(key: str, upload_id: str) -> list[dict]:
|
||||||
|
s = get_settings()
|
||||||
|
response = _client().list_parts(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
|
||||||
|
return response.get("Parts", [])
|
||||||
|
|
||||||
|
|
||||||
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
|
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
|
||||||
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
|
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
|
||||||
s = get_settings()
|
s = get_settings()
|
||||||
|
|
|
||||||
|
|
@ -71,6 +71,15 @@ async def init_db():
|
||||||
from src.core.security import hash_password
|
from src.core.security import hash_password
|
||||||
async with get_engine().begin() as conn:
|
async with get_engine().begin() as conn:
|
||||||
await conn.run_sync(Base.metadata.create_all)
|
await conn.run_sync(Base.metadata.create_all)
|
||||||
|
if conn.dialect.name == "postgresql":
|
||||||
|
await conn.execute(text(
|
||||||
|
"ALTER TABLE upload_items ADD COLUMN IF NOT EXISTS "
|
||||||
|
"multipart_upload_id VARCHAR(512)"
|
||||||
|
))
|
||||||
|
await conn.execute(text(
|
||||||
|
"CREATE INDEX IF NOT EXISTS ix_upload_items_multipart_upload_id "
|
||||||
|
"ON upload_items (multipart_upload_id)"
|
||||||
|
))
|
||||||
factory = get_session_factory()
|
factory = get_session_factory()
|
||||||
async with factory() as s:
|
async with factory() as s:
|
||||||
admin = (await s.execute(select(User).where(User.email == "admin@danagble.local"))).scalar_one_or_none()
|
admin = (await s.execute(select(User).where(User.email == "admin@danagble.local"))).scalar_one_or_none()
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,11 @@ async def create(project_id: str, body: CreateUploadJobIn,
|
||||||
return await svc.create_job(project_id, body.kind, [f.model_dump() for f in body.files], db)
|
return await svc.create_job(project_id, body.kind, [f.model_dump() for f in body.files], db)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("")
|
||||||
|
async def list_active(project_id: str, db: AsyncSession = Depends(get_session)):
|
||||||
|
return await svc.list_active(project_id, db)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{job_id}")
|
@router.get("/{job_id}")
|
||||||
async def get(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
async def get(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
||||||
data = await svc.get_job_detail(job_id, db)
|
data = await svc.get_job_detail(job_id, db)
|
||||||
|
|
|
||||||
|
|
@ -9,9 +9,10 @@ from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from models import Project
|
from models import Project, UploadItem, UploadJob
|
||||||
from shared import r2
|
from shared import r2
|
||||||
from src.core.db import get_session
|
from src.core.db import get_session
|
||||||
|
|
||||||
|
|
@ -64,7 +65,17 @@ async def start_multipart(project_id: str, body: MultipartStart,
|
||||||
upload_id = r2.create_multipart(key, body.contentType)
|
upload_id = r2.create_multipart(key, body.contentType)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(503, f"storage unreachable: {e}")
|
raise HTTPException(503, f"storage unreachable: {e}")
|
||||||
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
|
job = UploadJob(project_id=project_id, kind="zip", total_files=1)
|
||||||
|
db.add(job)
|
||||||
|
await db.flush()
|
||||||
|
item = UploadItem(project_id=project_id, job_id=job.id,
|
||||||
|
file_name=body.fileName,
|
||||||
|
content_type=body.contentType,
|
||||||
|
r2_key=key, multipart_upload_id=upload_id)
|
||||||
|
db.add(item)
|
||||||
|
await db.commit()
|
||||||
|
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE,
|
||||||
|
"jobId": job.id, "itemId": item.id}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/multipart/parts")
|
@router.post("/multipart/parts")
|
||||||
|
|
@ -91,6 +102,18 @@ async def complete_multipart(project_id: str, body: MultipartComplete,
|
||||||
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
|
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
|
||||||
if not r2.head_exists(body.key):
|
if not r2.head_exists(body.key):
|
||||||
raise HTTPException(400, "object not found after complete")
|
raise HTTPException(400, "object not found after complete")
|
||||||
|
item = (await db.execute(select(UploadItem).where(
|
||||||
|
UploadItem.project_id == project_id,
|
||||||
|
UploadItem.r2_key == body.key,
|
||||||
|
UploadItem.multipart_upload_id == body.uploadId
|
||||||
|
))).scalar_one_or_none()
|
||||||
|
if item is not None:
|
||||||
|
item.status = "uploaded"
|
||||||
|
job = await db.get(UploadJob, item.job_id)
|
||||||
|
if job is not None:
|
||||||
|
job.status = "uploaded"
|
||||||
|
job.uploaded_files = 1
|
||||||
|
await db.commit()
|
||||||
return {"key": body.key, "etag": etag}
|
return {"key": body.key, "etag": etag}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,14 @@ async def get_job(job_id: str, db: AsyncSession) -> dict:
|
||||||
return _job_dict(job)
|
return _job_dict(job)
|
||||||
|
|
||||||
|
|
||||||
|
async def list_active(project_id: str, db: AsyncSession) -> list[dict]:
|
||||||
|
rows = (await db.execute(select(UploadJob).where(
|
||||||
|
UploadJob.project_id == project_id,
|
||||||
|
UploadJob.status.in_(["uploading", "uploaded", "processing"]),
|
||||||
|
).order_by(UploadJob.created_at.desc()))).scalars().all()
|
||||||
|
return [_job_dict(job) for job in rows]
|
||||||
|
|
||||||
|
|
||||||
async def get_job_detail(job_id: str, db: AsyncSession) -> dict:
|
async def get_job_detail(job_id: str, db: AsyncSession) -> dict:
|
||||||
job = await db.get(UploadJob, job_id)
|
job = await db.get(UploadJob, job_id)
|
||||||
if job is None:
|
if job is None:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue