feat: add resumable dataset uploads

This commit is contained in:
hailt 2026-11-12 17:31:42 +07:00
parent f8ed08b854
commit 8d40cff227
23 changed files with 913 additions and 21 deletions

View file

@ -71,3 +71,11 @@ async def invite(body: InviteIn, db: AsyncSession = Depends(get_session)):
async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)):
from src.services import users as svc
return await svc.patch_user(user_id, body.model_dump(), db)
@users_router.delete("/{user_id}")
async def delete_user(user_id: str, db: AsyncSession = Depends(get_session),
user = Depends(get_current_user)):
from src.services import users as svc
actor = user.id if user is not None else None
return await svc.delete_user(user_id, actor, db)

View file

@ -3,13 +3,21 @@ from __future__ import annotations
from fastapi import APIRouter, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from shared.schemas import ImageConfirm, PresignRequest
from shared.schemas import BatchPresignItem, ImageConfirm, PresignRequest
from src.core.db import get_session
from src.services import images as svc
router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"])
@router.get("/by-id/{image_id}")
async def locate_image(project_id: str, image_id: str,
db: AsyncSession = Depends(get_session)):
"""Deep-link support (?imageId=): offset of one image in list order."""
from src.services import images as svc
return await svc.locate_image(project_id, image_id, db)
@router.get("")
async def list_images(project_id: str, status: str | None = Query(default=None),
search: str | None = Query(default=None),
@ -26,6 +34,13 @@ async def presign_upload(project_id: str, body: PresignRequest):
return svc.build_presign(project_id, body.fileName, body.contentType)
@router.post("/batch-presign")
async def batch_presign_upload(project_id: str, body: list[BatchPresignItem],
db: AsyncSession = Depends(get_session)):
"""One call for a whole upload batch (≤200 files)."""
return await svc.batch_presign(project_id, [b.model_dump() for b in body], db)
@router.post("", status_code=201)
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)):
return await svc.confirm_image(project_id, body, db)

42
src/api/v1/imports.py Normal file
View file

@ -0,0 +1,42 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from src.core.db import get_session
from src.services import imports as svc
router = APIRouter(prefix="/projects/{project_id}/imports", tags=["imports"])
class LocalZipIn(BaseModel):
path: str
class R2ZipIn(BaseModel):
r2_key: str
@router.post("/local-zip", status_code=202)
async def start_local_zip(project_id: str, body: LocalZipIn,
db: AsyncSession = Depends(get_session)):
"""Queue a server-side zip import (GB-scale, streamed). Poll the job id."""
return await svc.start_local_zip(project_id, body.path, db)
@router.post("/r2-zip", status_code=202)
async def start_r2_zip(project_id: str, body: R2ZipIn,
db: AsyncSession = Depends(get_session)):
"""Queue import of a zip already uploaded to R2 (e.g. via multipart)."""
return await svc.start_r2_zip(project_id, body.r2_key, db)
@router.get("")
async def list_imports(project_id: str, db: AsyncSession = Depends(get_session)):
return await svc.list_jobs(project_id, db)
@router.get("/{job_id}")
async def get_import(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
return await svc.get_job(project_id, job_id, db)

View file

@ -3,8 +3,9 @@ from __future__ import annotations
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from shared.schemas import LabelClassIn, ProjectCreate
from shared.schemas import AnnotationUpsert, BulkAnnotationItem, LabelClassIn, ProjectCreate
from src.core.db import get_session
from src.services import annotations as annotations_svc
from src.services import projects as svc
router = APIRouter(prefix="/projects", tags=["projects"])
@ -35,6 +36,14 @@ async def delete_project(project_id: str, db: AsyncSession = Depends(get_session
return await svc.delete_project(project_id, db)
@router.put("/{project_id}/annotations/batch")
async def bulk_upsert_annotations(project_id: str, body: dict[str, BulkAnnotationItem],
db: AsyncSession = Depends(get_session)):
"""COCO/YOLO import: upsert up to 500 images in one commit."""
items = {image_id: AnnotationUpsert(**item.model_dump()) for image_id, item in body.items()}
return await annotations_svc.bulk_upsert(project_id, items, db)
@router.post("/{project_id}/classes", status_code=201)
async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)):
return await svc.add_label_class(project_id, body, db)

View file

@ -1,6 +1,6 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
@ -22,6 +22,28 @@ async def metrics():
return get_snapshot()
_usage_cache: dict = {"at": 0.0, "data": None}
@router.get("/storage/usage")
async def storage_usage():
"""Real bucket bytes/objects for the sidebar meter (10-min cache)."""
import time
from shared import r2
from src.core.config import get_settings
now = time.time()
if _usage_cache["data"] is not None and now - _usage_cache["at"] < 600:
return _usage_cache["data"]
try:
usage = r2.bucket_usage()
except Exception as e:
raise HTTPException(503, f"storage unreachable: {e}")
data = {"bytes": usage["bytes"], "objects": usage["objects"],
"quota_bytes": get_settings().storage_quota_bytes}
_usage_cache.update({"at": now, "data": data})
return data
@router.post("/admin/backfill-thumbnails")
async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)):
"""Mint thumbs for images uploaded before thumbnailing existed. Dev-only."""

102
src/api/v1/uploads.py Normal file
View file

@ -0,0 +1,102 @@
"""Resumable multipart uploads: big single files (GB zip/TIFF/video).
Flow: POST /multipart -> {key, uploadId} -> PUT each part to its presigned
URL (parallel, retried per part, resumable via stored ETags) ->
POST /multipart/complete. Use for anything over ~100MB; small files keep
the single-PUT presign path.
"""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from models import Project
from shared import r2
from src.core.db import get_session
router = APIRouter(prefix="/projects/{project_id}/uploads", tags=["uploads"])
PART_SIZE = 8 * 1024 * 1024
class MultipartStart(BaseModel):
fileName: str
contentType: str = "application/octet-stream"
class MultipartParts(BaseModel):
key: str
uploadId: str
numbers: list[int] = Field(max_length=1000)
class PartETag(BaseModel):
ETag: str
PartNumber: int
class MultipartComplete(BaseModel):
key: str
uploadId: str
parts: list[PartETag]
class MultipartAbort(BaseModel):
key: str
uploadId: str
async def _project_slug(project_id: str, db: AsyncSession) -> str:
p = await db.get(Project, project_id)
if p is None:
raise HTTPException(404, "project not found")
return p.slug
@router.post("/multipart")
async def start_multipart(project_id: str, body: MultipartStart,
db: AsyncSession = Depends(get_session)):
import uuid as _uuid
slug = await _project_slug(project_id, db)
key = f"uploads/{slug}/{_uuid.uuid4().hex[:12]}-{body.fileName.replace('/', '_')}"
try:
upload_id = r2.create_multipart(key, body.contentType)
except Exception as e:
raise HTTPException(503, f"storage unreachable: {e}")
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
@router.post("/multipart/parts")
async def part_urls(project_id: str, body: MultipartParts,
db: AsyncSession = Depends(get_session)):
await _project_slug(project_id, db)
if any(n < 1 or n > 10000 for n in body.numbers):
raise HTTPException(400, "invalid part numbers")
try:
urls = {n: r2.presign_part(body.key, body.uploadId, n) for n in body.numbers}
except Exception as e:
raise HTTPException(503, f"storage unreachable: {e}")
return {"urls": urls}
@router.post("/multipart/complete")
async def complete_multipart(project_id: str, body: MultipartComplete,
db: AsyncSession = Depends(get_session)):
await _project_slug(project_id, db)
try:
etag = r2.complete_multipart(body.key, body.uploadId,
[p.model_dump() for p in body.parts])
except Exception as e:
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
if not r2.head_exists(body.key):
raise HTTPException(400, "object not found after complete")
return {"key": body.key, "etag": etag}
@router.post("/multipart/abort")
async def abort_multipart(project_id: str, body: MultipartAbort,
db: AsyncSession = Depends(get_session)):
await _project_slug(project_id, db)
r2.abort_multipart(body.key, body.uploadId)
return {"aborted": True}