diff --git a/models/__init__.py b/models/__init__.py index b755704..47d52a3 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -6,10 +6,12 @@ from models.tables import ( ExportJob, Image, ImportJob, + UploadItem, + UploadJob, LabelClass, Project, User, ) __all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image", - "ImportJob", "LabelClass", "Project", "User"] + "ImportJob", "LabelClass", "Project", "UploadItem", "UploadJob", "User"] diff --git a/models/tables.py b/models/tables.py index 97d2598..017a7aa 100644 --- a/models/tables.py +++ b/models/tables.py @@ -120,6 +120,37 @@ class ImportJob(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) +class UploadJob(Base): + """Durable browser upload batch; bytes live in object storage.""" + __tablename__ = "upload_jobs" + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid) + project_id: Mapped[str] = mapped_column(String(32), index=True) + kind: Mapped[str] = mapped_column(String(32), default="images") + status: Mapped[str] = mapped_column(String(32), default="uploading", index=True) + total_files: Mapped[int] = mapped_column(Integer, default=0) + uploaded_files: Mapped[int] = mapped_column(Integer, default=0) + processed_files: Mapped[int] = mapped_column(Integer, default=0) + failed_files: Mapped[int] = mapped_column(Integer, default=0) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class UploadItem(Base): + __tablename__ = "upload_items" + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid) + job_id: Mapped[str] = mapped_column(ForeignKey("upload_jobs.id", ondelete="CASCADE"), index=True) + project_id: Mapped[str] = mapped_column(String(32), index=True) + file_name: Mapped[str] = mapped_column(String(1024)) + content_type: Mapped[str] = mapped_column(String(255), default="application/octet-stream") + size: Mapped[int] = mapped_column(Integer, default=0) + r2_key: Mapped[str] = mapped_column(String(1024)) + status: Mapped[str] = mapped_column(String(32), default="pending", index=True) + image_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + class ExportJob(Base): __tablename__ = "exports" id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid) diff --git a/src/api/v1/upload_jobs.py b/src/api/v1/upload_jobs.py new file mode 100644 index 0000000..cda5e8f --- /dev/null +++ b/src/api/v1/upload_jobs.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import asyncio +import json + +from fastapi import APIRouter, Depends +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db import get_session, get_session_factory +from src.services import upload_jobs as svc + +router = APIRouter(prefix="/projects/{project_id}/upload-jobs", tags=["upload-jobs"]) + + +class UploadFileIn(BaseModel): + fileName: str + contentType: str = "application/octet-stream" + size: int = Field(default=0, ge=0) + + +class CreateUploadJobIn(BaseModel): + kind: str = "images" + files: list[UploadFileIn] = Field(min_length=1, max_length=2000) + + +class CompleteItemIn(BaseModel): + width: int = Field(default=0, ge=0) + height: int = Field(default=0, ge=0) + + +@router.post("", status_code=201) +async def create(project_id: str, body: CreateUploadJobIn, + db: AsyncSession = Depends(get_session)): + return await svc.create_job(project_id, body.kind, [f.model_dump() for f in body.files], db) + + +@router.get("/{job_id}") +async def get(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)): + data = await svc.get_job(job_id, db) + if data["projectId"] != project_id: + from fastapi import HTTPException + raise HTTPException(404, "upload job not found") + return data + + +@router.post("/{job_id}/items/{item_id}/complete") +async def complete(project_id: str, job_id: str, item_id: str, body: CompleteItemIn, + db: AsyncSession = Depends(get_session)): + data = await svc.complete_item(job_id, item_id, body.width, body.height, db) + if data["projectId"] != project_id: + from fastapi import HTTPException + raise HTTPException(404, "upload job not found") + return data + + +@router.get("/{job_id}/events") +async def events(project_id: str, job_id: str): + async def stream(): + factory = get_session_factory() + while True: + async with factory() as db: + data = await svc.get_job(job_id, db) + if data["projectId"] != project_id: + break + yield f"event: progress\ndata: {json.dumps(data)}\n\n" + if data["status"] in {"done", "partial_error", "error", "cancelled"}: + break + await asyncio.sleep(1) + return StreamingResponse(stream(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) diff --git a/src/main.py b/src/main.py index 3e9ff4b..164f15a 100644 --- a/src/main.py +++ b/src/main.py @@ -11,7 +11,7 @@ from shared import events from shared.logger import get_logger from shared.tracing import TraceMiddleware from shared.traffic import TrafficMiddleware -from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads +from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, upload_jobs, uploads from src.api.v1.auth import users_router from src.core.config import get_settings from src.core.security import get_current_user @@ -63,6 +63,7 @@ app.include_router(images.router, prefix=prefix, dependencies=authed) app.include_router(annotations.router, prefix=prefix, dependencies=authed) app.include_router(imports.router, prefix=prefix, dependencies=authed) app.include_router(uploads.router, prefix=prefix, dependencies=authed) +app.include_router(upload_jobs.router, prefix=prefix, dependencies=authed) @app.get("/") diff --git a/src/services/upload_jobs.py b/src/services/upload_jobs.py new file mode 100644 index 0000000..c9c0214 --- /dev/null +++ b/src/services/upload_jobs.py @@ -0,0 +1,88 @@ +"""Durable upload batches: object bytes go to MinIO, metadata stays in Postgres.""" +from __future__ import annotations + +import uuid + +from fastapi import HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from models import Image, Project, UploadItem, UploadJob +from shared import keys, r2 + + +def _item_dict(item: UploadItem, put_url: str | None = None) -> dict: + out = {"id": item.id, "fileName": item.file_name, "status": item.status, + "size": item.size, "r2Key": item.r2_key, "imageId": item.image_id, + "error": item.error} + if put_url: + out["putUrl"] = put_url + return out + + +def _job_dict(job: UploadJob) -> dict: + return {"id": job.id, "projectId": job.project_id, "kind": job.kind, + "status": job.status, "totalFiles": job.total_files, + "uploadedFiles": job.uploaded_files, "processedFiles": job.processed_files, + "failedFiles": job.failed_files, "error": job.error} + + +async def create_job(project_id: str, kind: str, files: list[dict], db: AsyncSession) -> dict: + if await db.get(Project, project_id) is None: + raise HTTPException(404, "project not found") + if not files or len(files) > 2000: + raise HTTPException(400, "files must contain 1..2000 items") + job = UploadJob(id=uuid.uuid4().hex[:12], project_id=project_id, + kind=kind or "images", total_files=len(files)) + db.add(job) + items = [] + for f in files: + item_id = uuid.uuid4().hex[:12] + name = str(f.get("fileName") or "upload.bin") + content_type = str(f.get("contentType") or "application/octet-stream") + item = UploadItem(id=item_id, job_id=job.id, project_id=project_id, + file_name=name, content_type=content_type, + size=max(0, int(f.get("size") or 0)), + r2_key=keys.image_key(project_id, item_id, name)) + db.add(item) + items.append(item) + await db.commit() + return {"job": _job_dict(job), "items": [_item_dict( + item, r2.presign_put(item.r2_key, item.content_type)) for item in items]} + + +async def get_job(job_id: str, db: AsyncSession) -> dict: + job = await db.get(UploadJob, job_id) + if job is None: + raise HTTPException(404, "upload job not found") + return _job_dict(job) + + +async def complete_item(job_id: str, item_id: str, width: int, height: int, + db: AsyncSession) -> dict: + item = await db.get(UploadItem, item_id) + if item is None or item.job_id != job_id: + raise HTTPException(404, "upload item not found") + job = await db.get(UploadJob, job_id) + if job is None: + raise HTTPException(404, "upload job not found") + if item.status == "done": + return {**_job_dict(job), "imageId": item.image_id} + if not r2.head_exists(item.r2_key): + item.status = "error" + item.error = "object not found in storage" + job.failed_files += 1 + await db.commit() + raise HTTPException(400, item.error) + image = Image(project_id=item.project_id, file_name=item.file_name, + r2_key=item.r2_key, width=width, height=height, + status="unassigned", flags={}) + db.add(image) + item.status = "done" + item.image_id = image.id + job.uploaded_files += 1 + job.processed_files += 1 + if job.processed_files + job.failed_files >= job.total_files: + job.status = "done" if job.failed_files == 0 else "partial_error" + await db.commit() + return {**_job_dict(job), "imageId": image.id}