feat: add durable upload job API and SSE progress

This commit is contained in:
hailt 2026-11-12 22:40:24 +07:00
parent 2b8f8551ce
commit cf1e448a65
5 changed files with 196 additions and 2 deletions

View file

@ -6,10 +6,12 @@ from models.tables import (
ExportJob, ExportJob,
Image, Image,
ImportJob, ImportJob,
UploadItem,
UploadJob,
LabelClass, LabelClass,
Project, Project,
User, User,
) )
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image", __all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
"ImportJob", "LabelClass", "Project", "User"] "ImportJob", "LabelClass", "Project", "UploadItem", "UploadJob", "User"]

View file

@ -120,6 +120,37 @@ class ImportJob(Base):
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class UploadJob(Base):
"""Durable browser upload batch; bytes live in object storage."""
__tablename__ = "upload_jobs"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
project_id: Mapped[str] = mapped_column(String(32), index=True)
kind: Mapped[str] = mapped_column(String(32), default="images")
status: Mapped[str] = mapped_column(String(32), default="uploading", index=True)
total_files: Mapped[int] = mapped_column(Integer, default=0)
uploaded_files: Mapped[int] = mapped_column(Integer, default=0)
processed_files: Mapped[int] = mapped_column(Integer, default=0)
failed_files: Mapped[int] = mapped_column(Integer, default=0)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class UploadItem(Base):
__tablename__ = "upload_items"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
job_id: Mapped[str] = mapped_column(ForeignKey("upload_jobs.id", ondelete="CASCADE"), index=True)
project_id: Mapped[str] = mapped_column(String(32), index=True)
file_name: Mapped[str] = mapped_column(String(1024))
content_type: Mapped[str] = mapped_column(String(255), default="application/octet-stream")
size: Mapped[int] = mapped_column(Integer, default=0)
r2_key: Mapped[str] = mapped_column(String(1024))
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class ExportJob(Base): class ExportJob(Base):
__tablename__ = "exports" __tablename__ = "exports"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)

72
src/api/v1/upload_jobs.py Normal file
View file

@ -0,0 +1,72 @@
from __future__ import annotations
import asyncio
import json
from fastapi import APIRouter, Depends
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from src.core.db import get_session, get_session_factory
from src.services import upload_jobs as svc
router = APIRouter(prefix="/projects/{project_id}/upload-jobs", tags=["upload-jobs"])
class UploadFileIn(BaseModel):
fileName: str
contentType: str = "application/octet-stream"
size: int = Field(default=0, ge=0)
class CreateUploadJobIn(BaseModel):
kind: str = "images"
files: list[UploadFileIn] = Field(min_length=1, max_length=2000)
class CompleteItemIn(BaseModel):
width: int = Field(default=0, ge=0)
height: int = Field(default=0, ge=0)
@router.post("", status_code=201)
async def create(project_id: str, body: CreateUploadJobIn,
db: AsyncSession = Depends(get_session)):
return await svc.create_job(project_id, body.kind, [f.model_dump() for f in body.files], db)
@router.get("/{job_id}")
async def get(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
data = await svc.get_job(job_id, db)
if data["projectId"] != project_id:
from fastapi import HTTPException
raise HTTPException(404, "upload job not found")
return data
@router.post("/{job_id}/items/{item_id}/complete")
async def complete(project_id: str, job_id: str, item_id: str, body: CompleteItemIn,
db: AsyncSession = Depends(get_session)):
data = await svc.complete_item(job_id, item_id, body.width, body.height, db)
if data["projectId"] != project_id:
from fastapi import HTTPException
raise HTTPException(404, "upload job not found")
return data
@router.get("/{job_id}/events")
async def events(project_id: str, job_id: str):
async def stream():
factory = get_session_factory()
while True:
async with factory() as db:
data = await svc.get_job(job_id, db)
if data["projectId"] != project_id:
break
yield f"event: progress\ndata: {json.dumps(data)}\n\n"
if data["status"] in {"done", "partial_error", "error", "cancelled"}:
break
await asyncio.sleep(1)
return StreamingResponse(stream(), media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})

View file

@ -11,7 +11,7 @@ from shared import events
from shared.logger import get_logger from shared.logger import get_logger
from shared.tracing import TraceMiddleware from shared.tracing import TraceMiddleware
from shared.traffic import TrafficMiddleware from shared.traffic import TrafficMiddleware
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, upload_jobs, uploads
from src.api.v1.auth import users_router from src.api.v1.auth import users_router
from src.core.config import get_settings from src.core.config import get_settings
from src.core.security import get_current_user from src.core.security import get_current_user
@ -63,6 +63,7 @@ app.include_router(images.router, prefix=prefix, dependencies=authed)
app.include_router(annotations.router, prefix=prefix, dependencies=authed) app.include_router(annotations.router, prefix=prefix, dependencies=authed)
app.include_router(imports.router, prefix=prefix, dependencies=authed) app.include_router(imports.router, prefix=prefix, dependencies=authed)
app.include_router(uploads.router, prefix=prefix, dependencies=authed) app.include_router(uploads.router, prefix=prefix, dependencies=authed)
app.include_router(upload_jobs.router, prefix=prefix, dependencies=authed)
@app.get("/") @app.get("/")

View file

@ -0,0 +1,88 @@
"""Durable upload batches: object bytes go to MinIO, metadata stays in Postgres."""
from __future__ import annotations
import uuid
from fastapi import HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models import Image, Project, UploadItem, UploadJob
from shared import keys, r2
def _item_dict(item: UploadItem, put_url: str | None = None) -> dict:
out = {"id": item.id, "fileName": item.file_name, "status": item.status,
"size": item.size, "r2Key": item.r2_key, "imageId": item.image_id,
"error": item.error}
if put_url:
out["putUrl"] = put_url
return out
def _job_dict(job: UploadJob) -> dict:
return {"id": job.id, "projectId": job.project_id, "kind": job.kind,
"status": job.status, "totalFiles": job.total_files,
"uploadedFiles": job.uploaded_files, "processedFiles": job.processed_files,
"failedFiles": job.failed_files, "error": job.error}
async def create_job(project_id: str, kind: str, files: list[dict], db: AsyncSession) -> dict:
if await db.get(Project, project_id) is None:
raise HTTPException(404, "project not found")
if not files or len(files) > 2000:
raise HTTPException(400, "files must contain 1..2000 items")
job = UploadJob(id=uuid.uuid4().hex[:12], project_id=project_id,
kind=kind or "images", total_files=len(files))
db.add(job)
items = []
for f in files:
item_id = uuid.uuid4().hex[:12]
name = str(f.get("fileName") or "upload.bin")
content_type = str(f.get("contentType") or "application/octet-stream")
item = UploadItem(id=item_id, job_id=job.id, project_id=project_id,
file_name=name, content_type=content_type,
size=max(0, int(f.get("size") or 0)),
r2_key=keys.image_key(project_id, item_id, name))
db.add(item)
items.append(item)
await db.commit()
return {"job": _job_dict(job), "items": [_item_dict(
item, r2.presign_put(item.r2_key, item.content_type)) for item in items]}
async def get_job(job_id: str, db: AsyncSession) -> dict:
job = await db.get(UploadJob, job_id)
if job is None:
raise HTTPException(404, "upload job not found")
return _job_dict(job)
async def complete_item(job_id: str, item_id: str, width: int, height: int,
db: AsyncSession) -> dict:
item = await db.get(UploadItem, item_id)
if item is None or item.job_id != job_id:
raise HTTPException(404, "upload item not found")
job = await db.get(UploadJob, job_id)
if job is None:
raise HTTPException(404, "upload job not found")
if item.status == "done":
return {**_job_dict(job), "imageId": item.image_id}
if not r2.head_exists(item.r2_key):
item.status = "error"
item.error = "object not found in storage"
job.failed_files += 1
await db.commit()
raise HTTPException(400, item.error)
image = Image(project_id=item.project_id, file_name=item.file_name,
r2_key=item.r2_key, width=width, height=height,
status="unassigned", flags={})
db.add(image)
item.status = "done"
item.image_id = image.id
job.uploaded_files += 1
job.processed_files += 1
if job.processed_files + job.failed_files >= job.total_files:
job.status = "done" if job.failed_files == 0 else "partial_error"
await db.commit()
return {**_job_dict(job), "imageId": image.id}