feat: add durable upload job API and SSE progress
This commit is contained in:
parent
2b8f8551ce
commit
cf1e448a65
5 changed files with 196 additions and 2 deletions
|
|
@ -6,10 +6,12 @@ from models.tables import (
|
||||||
ExportJob,
|
ExportJob,
|
||||||
Image,
|
Image,
|
||||||
ImportJob,
|
ImportJob,
|
||||||
|
UploadItem,
|
||||||
|
UploadJob,
|
||||||
LabelClass,
|
LabelClass,
|
||||||
Project,
|
Project,
|
||||||
User,
|
User,
|
||||||
)
|
)
|
||||||
|
|
||||||
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
|
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
|
||||||
"ImportJob", "LabelClass", "Project", "User"]
|
"ImportJob", "LabelClass", "Project", "UploadItem", "UploadJob", "User"]
|
||||||
|
|
|
||||||
|
|
@ -120,6 +120,37 @@ class ImportJob(Base):
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
|
||||||
|
class UploadJob(Base):
|
||||||
|
"""Durable browser upload batch; bytes live in object storage."""
|
||||||
|
__tablename__ = "upload_jobs"
|
||||||
|
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||||
|
project_id: Mapped[str] = mapped_column(String(32), index=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(32), default="images")
|
||||||
|
status: Mapped[str] = mapped_column(String(32), default="uploading", index=True)
|
||||||
|
total_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
uploaded_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
processed_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
failed_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
|
||||||
|
class UploadItem(Base):
|
||||||
|
__tablename__ = "upload_items"
|
||||||
|
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||||
|
job_id: Mapped[str] = mapped_column(ForeignKey("upload_jobs.id", ondelete="CASCADE"), index=True)
|
||||||
|
project_id: Mapped[str] = mapped_column(String(32), index=True)
|
||||||
|
file_name: Mapped[str] = mapped_column(String(1024))
|
||||||
|
content_type: Mapped[str] = mapped_column(String(255), default="application/octet-stream")
|
||||||
|
size: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
r2_key: Mapped[str] = mapped_column(String(1024))
|
||||||
|
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||||
|
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
|
||||||
class ExportJob(Base):
|
class ExportJob(Base):
|
||||||
__tablename__ = "exports"
|
__tablename__ = "exports"
|
||||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||||
|
|
|
||||||
72
src/api/v1/upload_jobs.py
Normal file
72
src/api/v1/upload_jobs.py
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from src.core.db import get_session, get_session_factory
|
||||||
|
from src.services import upload_jobs as svc
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/projects/{project_id}/upload-jobs", tags=["upload-jobs"])
|
||||||
|
|
||||||
|
|
||||||
|
class UploadFileIn(BaseModel):
|
||||||
|
fileName: str
|
||||||
|
contentType: str = "application/octet-stream"
|
||||||
|
size: int = Field(default=0, ge=0)
|
||||||
|
|
||||||
|
|
||||||
|
class CreateUploadJobIn(BaseModel):
|
||||||
|
kind: str = "images"
|
||||||
|
files: list[UploadFileIn] = Field(min_length=1, max_length=2000)
|
||||||
|
|
||||||
|
|
||||||
|
class CompleteItemIn(BaseModel):
|
||||||
|
width: int = Field(default=0, ge=0)
|
||||||
|
height: int = Field(default=0, ge=0)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", status_code=201)
|
||||||
|
async def create(project_id: str, body: CreateUploadJobIn,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
return await svc.create_job(project_id, body.kind, [f.model_dump() for f in body.files], db)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{job_id}")
|
||||||
|
async def get(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
||||||
|
data = await svc.get_job(job_id, db)
|
||||||
|
if data["projectId"] != project_id:
|
||||||
|
from fastapi import HTTPException
|
||||||
|
raise HTTPException(404, "upload job not found")
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{job_id}/items/{item_id}/complete")
|
||||||
|
async def complete(project_id: str, job_id: str, item_id: str, body: CompleteItemIn,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
data = await svc.complete_item(job_id, item_id, body.width, body.height, db)
|
||||||
|
if data["projectId"] != project_id:
|
||||||
|
from fastapi import HTTPException
|
||||||
|
raise HTTPException(404, "upload job not found")
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{job_id}/events")
|
||||||
|
async def events(project_id: str, job_id: str):
|
||||||
|
async def stream():
|
||||||
|
factory = get_session_factory()
|
||||||
|
while True:
|
||||||
|
async with factory() as db:
|
||||||
|
data = await svc.get_job(job_id, db)
|
||||||
|
if data["projectId"] != project_id:
|
||||||
|
break
|
||||||
|
yield f"event: progress\ndata: {json.dumps(data)}\n\n"
|
||||||
|
if data["status"] in {"done", "partial_error", "error", "cancelled"}:
|
||||||
|
break
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
return StreamingResponse(stream(), media_type="text/event-stream",
|
||||||
|
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
||||||
|
|
@ -11,7 +11,7 @@ from shared import events
|
||||||
from shared.logger import get_logger
|
from shared.logger import get_logger
|
||||||
from shared.tracing import TraceMiddleware
|
from shared.tracing import TraceMiddleware
|
||||||
from shared.traffic import TrafficMiddleware
|
from shared.traffic import TrafficMiddleware
|
||||||
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads
|
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, upload_jobs, uploads
|
||||||
from src.api.v1.auth import users_router
|
from src.api.v1.auth import users_router
|
||||||
from src.core.config import get_settings
|
from src.core.config import get_settings
|
||||||
from src.core.security import get_current_user
|
from src.core.security import get_current_user
|
||||||
|
|
@ -63,6 +63,7 @@ app.include_router(images.router, prefix=prefix, dependencies=authed)
|
||||||
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
|
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
|
||||||
app.include_router(imports.router, prefix=prefix, dependencies=authed)
|
app.include_router(imports.router, prefix=prefix, dependencies=authed)
|
||||||
app.include_router(uploads.router, prefix=prefix, dependencies=authed)
|
app.include_router(uploads.router, prefix=prefix, dependencies=authed)
|
||||||
|
app.include_router(upload_jobs.router, prefix=prefix, dependencies=authed)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
|
|
|
||||||
88
src/services/upload_jobs.py
Normal file
88
src/services/upload_jobs.py
Normal file
|
|
@ -0,0 +1,88 @@
|
||||||
|
"""Durable upload batches: object bytes go to MinIO, metadata stays in Postgres."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from models import Image, Project, UploadItem, UploadJob
|
||||||
|
from shared import keys, r2
|
||||||
|
|
||||||
|
|
||||||
|
def _item_dict(item: UploadItem, put_url: str | None = None) -> dict:
|
||||||
|
out = {"id": item.id, "fileName": item.file_name, "status": item.status,
|
||||||
|
"size": item.size, "r2Key": item.r2_key, "imageId": item.image_id,
|
||||||
|
"error": item.error}
|
||||||
|
if put_url:
|
||||||
|
out["putUrl"] = put_url
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _job_dict(job: UploadJob) -> dict:
|
||||||
|
return {"id": job.id, "projectId": job.project_id, "kind": job.kind,
|
||||||
|
"status": job.status, "totalFiles": job.total_files,
|
||||||
|
"uploadedFiles": job.uploaded_files, "processedFiles": job.processed_files,
|
||||||
|
"failedFiles": job.failed_files, "error": job.error}
|
||||||
|
|
||||||
|
|
||||||
|
async def create_job(project_id: str, kind: str, files: list[dict], db: AsyncSession) -> dict:
|
||||||
|
if await db.get(Project, project_id) is None:
|
||||||
|
raise HTTPException(404, "project not found")
|
||||||
|
if not files or len(files) > 2000:
|
||||||
|
raise HTTPException(400, "files must contain 1..2000 items")
|
||||||
|
job = UploadJob(id=uuid.uuid4().hex[:12], project_id=project_id,
|
||||||
|
kind=kind or "images", total_files=len(files))
|
||||||
|
db.add(job)
|
||||||
|
items = []
|
||||||
|
for f in files:
|
||||||
|
item_id = uuid.uuid4().hex[:12]
|
||||||
|
name = str(f.get("fileName") or "upload.bin")
|
||||||
|
content_type = str(f.get("contentType") or "application/octet-stream")
|
||||||
|
item = UploadItem(id=item_id, job_id=job.id, project_id=project_id,
|
||||||
|
file_name=name, content_type=content_type,
|
||||||
|
size=max(0, int(f.get("size") or 0)),
|
||||||
|
r2_key=keys.image_key(project_id, item_id, name))
|
||||||
|
db.add(item)
|
||||||
|
items.append(item)
|
||||||
|
await db.commit()
|
||||||
|
return {"job": _job_dict(job), "items": [_item_dict(
|
||||||
|
item, r2.presign_put(item.r2_key, item.content_type)) for item in items]}
|
||||||
|
|
||||||
|
|
||||||
|
async def get_job(job_id: str, db: AsyncSession) -> dict:
|
||||||
|
job = await db.get(UploadJob, job_id)
|
||||||
|
if job is None:
|
||||||
|
raise HTTPException(404, "upload job not found")
|
||||||
|
return _job_dict(job)
|
||||||
|
|
||||||
|
|
||||||
|
async def complete_item(job_id: str, item_id: str, width: int, height: int,
|
||||||
|
db: AsyncSession) -> dict:
|
||||||
|
item = await db.get(UploadItem, item_id)
|
||||||
|
if item is None or item.job_id != job_id:
|
||||||
|
raise HTTPException(404, "upload item not found")
|
||||||
|
job = await db.get(UploadJob, job_id)
|
||||||
|
if job is None:
|
||||||
|
raise HTTPException(404, "upload job not found")
|
||||||
|
if item.status == "done":
|
||||||
|
return {**_job_dict(job), "imageId": item.image_id}
|
||||||
|
if not r2.head_exists(item.r2_key):
|
||||||
|
item.status = "error"
|
||||||
|
item.error = "object not found in storage"
|
||||||
|
job.failed_files += 1
|
||||||
|
await db.commit()
|
||||||
|
raise HTTPException(400, item.error)
|
||||||
|
image = Image(project_id=item.project_id, file_name=item.file_name,
|
||||||
|
r2_key=item.r2_key, width=width, height=height,
|
||||||
|
status="unassigned", flags={})
|
||||||
|
db.add(image)
|
||||||
|
item.status = "done"
|
||||||
|
item.image_id = image.id
|
||||||
|
job.uploaded_files += 1
|
||||||
|
job.processed_files += 1
|
||||||
|
if job.processed_files + job.failed_files >= job.total_files:
|
||||||
|
job.status = "done" if job.failed_files == 0 else "partial_error"
|
||||||
|
await db.commit()
|
||||||
|
return {**_job_dict(job), "imageId": image.id}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue