feat: add durable upload job API and SSE progress
This commit is contained in:
parent
2b8f8551ce
commit
cf1e448a65
5 changed files with 196 additions and 2 deletions
|
|
@ -6,10 +6,12 @@ from models.tables import (
|
|||
ExportJob,
|
||||
Image,
|
||||
ImportJob,
|
||||
UploadItem,
|
||||
UploadJob,
|
||||
LabelClass,
|
||||
Project,
|
||||
User,
|
||||
)
|
||||
|
||||
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
|
||||
"ImportJob", "LabelClass", "Project", "User"]
|
||||
"ImportJob", "LabelClass", "Project", "UploadItem", "UploadJob", "User"]
|
||||
|
|
|
|||
|
|
@ -120,6 +120,37 @@ class ImportJob(Base):
|
|||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class UploadJob(Base):
|
||||
"""Durable browser upload batch; bytes live in object storage."""
|
||||
__tablename__ = "upload_jobs"
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||
project_id: Mapped[str] = mapped_column(String(32), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), default="images")
|
||||
status: Mapped[str] = mapped_column(String(32), default="uploading", index=True)
|
||||
total_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||
uploaded_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||
processed_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||
failed_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class UploadItem(Base):
|
||||
__tablename__ = "upload_items"
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||
job_id: Mapped[str] = mapped_column(ForeignKey("upload_jobs.id", ondelete="CASCADE"), index=True)
|
||||
project_id: Mapped[str] = mapped_column(String(32), index=True)
|
||||
file_name: Mapped[str] = mapped_column(String(1024))
|
||||
content_type: Mapped[str] = mapped_column(String(255), default="application/octet-stream")
|
||||
size: Mapped[int] = mapped_column(Integer, default=0)
|
||||
r2_key: Mapped[str] = mapped_column(String(1024))
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class ExportJob(Base):
|
||||
__tablename__ = "exports"
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||
|
|
|
|||
72
src/api/v1/upload_jobs.py
Normal file
72
src/api/v1/upload_jobs.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.core.db import get_session, get_session_factory
|
||||
from src.services import upload_jobs as svc
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/upload-jobs", tags=["upload-jobs"])
|
||||
|
||||
|
||||
class UploadFileIn(BaseModel):
|
||||
fileName: str
|
||||
contentType: str = "application/octet-stream"
|
||||
size: int = Field(default=0, ge=0)
|
||||
|
||||
|
||||
class CreateUploadJobIn(BaseModel):
|
||||
kind: str = "images"
|
||||
files: list[UploadFileIn] = Field(min_length=1, max_length=2000)
|
||||
|
||||
|
||||
class CompleteItemIn(BaseModel):
|
||||
width: int = Field(default=0, ge=0)
|
||||
height: int = Field(default=0, ge=0)
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create(project_id: str, body: CreateUploadJobIn,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
return await svc.create_job(project_id, body.kind, [f.model_dump() for f in body.files], db)
|
||||
|
||||
|
||||
@router.get("/{job_id}")
|
||||
async def get(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
||||
data = await svc.get_job(job_id, db)
|
||||
if data["projectId"] != project_id:
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(404, "upload job not found")
|
||||
return data
|
||||
|
||||
|
||||
@router.post("/{job_id}/items/{item_id}/complete")
|
||||
async def complete(project_id: str, job_id: str, item_id: str, body: CompleteItemIn,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
data = await svc.complete_item(job_id, item_id, body.width, body.height, db)
|
||||
if data["projectId"] != project_id:
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(404, "upload job not found")
|
||||
return data
|
||||
|
||||
|
||||
@router.get("/{job_id}/events")
|
||||
async def events(project_id: str, job_id: str):
|
||||
async def stream():
|
||||
factory = get_session_factory()
|
||||
while True:
|
||||
async with factory() as db:
|
||||
data = await svc.get_job(job_id, db)
|
||||
if data["projectId"] != project_id:
|
||||
break
|
||||
yield f"event: progress\ndata: {json.dumps(data)}\n\n"
|
||||
if data["status"] in {"done", "partial_error", "error", "cancelled"}:
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
return StreamingResponse(stream(), media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
||||
|
|
@ -11,7 +11,7 @@ from shared import events
|
|||
from shared.logger import get_logger
|
||||
from shared.tracing import TraceMiddleware
|
||||
from shared.traffic import TrafficMiddleware
|
||||
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads
|
||||
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, upload_jobs, uploads
|
||||
from src.api.v1.auth import users_router
|
||||
from src.core.config import get_settings
|
||||
from src.core.security import get_current_user
|
||||
|
|
@ -63,6 +63,7 @@ app.include_router(images.router, prefix=prefix, dependencies=authed)
|
|||
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(imports.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(uploads.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(upload_jobs.router, prefix=prefix, dependencies=authed)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
|
|
|
|||
88
src/services/upload_jobs.py
Normal file
88
src/services/upload_jobs.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
"""Durable upload batches: object bytes go to MinIO, metadata stays in Postgres."""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Image, Project, UploadItem, UploadJob
|
||||
from shared import keys, r2
|
||||
|
||||
|
||||
def _item_dict(item: UploadItem, put_url: str | None = None) -> dict:
|
||||
out = {"id": item.id, "fileName": item.file_name, "status": item.status,
|
||||
"size": item.size, "r2Key": item.r2_key, "imageId": item.image_id,
|
||||
"error": item.error}
|
||||
if put_url:
|
||||
out["putUrl"] = put_url
|
||||
return out
|
||||
|
||||
|
||||
def _job_dict(job: UploadJob) -> dict:
|
||||
return {"id": job.id, "projectId": job.project_id, "kind": job.kind,
|
||||
"status": job.status, "totalFiles": job.total_files,
|
||||
"uploadedFiles": job.uploaded_files, "processedFiles": job.processed_files,
|
||||
"failedFiles": job.failed_files, "error": job.error}
|
||||
|
||||
|
||||
async def create_job(project_id: str, kind: str, files: list[dict], db: AsyncSession) -> dict:
|
||||
if await db.get(Project, project_id) is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
if not files or len(files) > 2000:
|
||||
raise HTTPException(400, "files must contain 1..2000 items")
|
||||
job = UploadJob(id=uuid.uuid4().hex[:12], project_id=project_id,
|
||||
kind=kind or "images", total_files=len(files))
|
||||
db.add(job)
|
||||
items = []
|
||||
for f in files:
|
||||
item_id = uuid.uuid4().hex[:12]
|
||||
name = str(f.get("fileName") or "upload.bin")
|
||||
content_type = str(f.get("contentType") or "application/octet-stream")
|
||||
item = UploadItem(id=item_id, job_id=job.id, project_id=project_id,
|
||||
file_name=name, content_type=content_type,
|
||||
size=max(0, int(f.get("size") or 0)),
|
||||
r2_key=keys.image_key(project_id, item_id, name))
|
||||
db.add(item)
|
||||
items.append(item)
|
||||
await db.commit()
|
||||
return {"job": _job_dict(job), "items": [_item_dict(
|
||||
item, r2.presign_put(item.r2_key, item.content_type)) for item in items]}
|
||||
|
||||
|
||||
async def get_job(job_id: str, db: AsyncSession) -> dict:
|
||||
job = await db.get(UploadJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(404, "upload job not found")
|
||||
return _job_dict(job)
|
||||
|
||||
|
||||
async def complete_item(job_id: str, item_id: str, width: int, height: int,
|
||||
db: AsyncSession) -> dict:
|
||||
item = await db.get(UploadItem, item_id)
|
||||
if item is None or item.job_id != job_id:
|
||||
raise HTTPException(404, "upload item not found")
|
||||
job = await db.get(UploadJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(404, "upload job not found")
|
||||
if item.status == "done":
|
||||
return {**_job_dict(job), "imageId": item.image_id}
|
||||
if not r2.head_exists(item.r2_key):
|
||||
item.status = "error"
|
||||
item.error = "object not found in storage"
|
||||
job.failed_files += 1
|
||||
await db.commit()
|
||||
raise HTTPException(400, item.error)
|
||||
image = Image(project_id=item.project_id, file_name=item.file_name,
|
||||
r2_key=item.r2_key, width=width, height=height,
|
||||
status="unassigned", flags={})
|
||||
db.add(image)
|
||||
item.status = "done"
|
||||
item.image_id = image.id
|
||||
job.uploaded_files += 1
|
||||
job.processed_files += 1
|
||||
if job.processed_files + job.failed_files >= job.total_files:
|
||||
job.status = "done" if job.failed_files == 0 else "partial_error"
|
||||
await db.commit()
|
||||
return {**_job_dict(job), "imageId": image.id}
|
||||
Loading…
Add table
Add a link
Reference in a new issue