feat: add resumable dataset uploads
This commit is contained in:
parent
f8ed08b854
commit
8d40cff227
23 changed files with 913 additions and 21 deletions
|
|
@ -71,3 +71,11 @@ async def invite(body: InviteIn, db: AsyncSession = Depends(get_session)):
|
|||
async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)):
|
||||
from src.services import users as svc
|
||||
return await svc.patch_user(user_id, body.model_dump(), db)
|
||||
|
||||
|
||||
@users_router.delete("/{user_id}")
|
||||
async def delete_user(user_id: str, db: AsyncSession = Depends(get_session),
|
||||
user = Depends(get_current_user)):
|
||||
from src.services import users as svc
|
||||
actor = user.id if user is not None else None
|
||||
return await svc.delete_user(user_id, actor, db)
|
||||
|
|
|
|||
|
|
@ -3,13 +3,21 @@ from __future__ import annotations
|
|||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from shared.schemas import ImageConfirm, PresignRequest
|
||||
from shared.schemas import BatchPresignItem, ImageConfirm, PresignRequest
|
||||
from src.core.db import get_session
|
||||
from src.services import images as svc
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"])
|
||||
|
||||
|
||||
@router.get("/by-id/{image_id}")
|
||||
async def locate_image(project_id: str, image_id: str,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Deep-link support (?imageId=): offset of one image in list order."""
|
||||
from src.services import images as svc
|
||||
return await svc.locate_image(project_id, image_id, db)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_images(project_id: str, status: str | None = Query(default=None),
|
||||
search: str | None = Query(default=None),
|
||||
|
|
@ -26,6 +34,13 @@ async def presign_upload(project_id: str, body: PresignRequest):
|
|||
return svc.build_presign(project_id, body.fileName, body.contentType)
|
||||
|
||||
|
||||
@router.post("/batch-presign")
|
||||
async def batch_presign_upload(project_id: str, body: list[BatchPresignItem],
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""One call for a whole upload batch (≤200 files)."""
|
||||
return await svc.batch_presign(project_id, [b.model_dump() for b in body], db)
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.confirm_image(project_id, body, db)
|
||||
|
|
|
|||
42
src/api/v1/imports.py
Normal file
42
src/api/v1/imports.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.core.db import get_session
|
||||
from src.services import imports as svc
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/imports", tags=["imports"])
|
||||
|
||||
|
||||
class LocalZipIn(BaseModel):
|
||||
path: str
|
||||
|
||||
|
||||
class R2ZipIn(BaseModel):
|
||||
r2_key: str
|
||||
|
||||
|
||||
@router.post("/local-zip", status_code=202)
|
||||
async def start_local_zip(project_id: str, body: LocalZipIn,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Queue a server-side zip import (GB-scale, streamed). Poll the job id."""
|
||||
return await svc.start_local_zip(project_id, body.path, db)
|
||||
|
||||
|
||||
@router.post("/r2-zip", status_code=202)
|
||||
async def start_r2_zip(project_id: str, body: R2ZipIn,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Queue import of a zip already uploaded to R2 (e.g. via multipart)."""
|
||||
return await svc.start_r2_zip(project_id, body.r2_key, db)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_imports(project_id: str, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.list_jobs(project_id, db)
|
||||
|
||||
|
||||
@router.get("/{job_id}")
|
||||
async def get_import(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.get_job(project_id, job_id, db)
|
||||
|
|
@ -3,8 +3,9 @@ from __future__ import annotations
|
|||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from shared.schemas import LabelClassIn, ProjectCreate
|
||||
from shared.schemas import AnnotationUpsert, BulkAnnotationItem, LabelClassIn, ProjectCreate
|
||||
from src.core.db import get_session
|
||||
from src.services import annotations as annotations_svc
|
||||
from src.services import projects as svc
|
||||
|
||||
router = APIRouter(prefix="/projects", tags=["projects"])
|
||||
|
|
@ -35,6 +36,14 @@ async def delete_project(project_id: str, db: AsyncSession = Depends(get_session
|
|||
return await svc.delete_project(project_id, db)
|
||||
|
||||
|
||||
@router.put("/{project_id}/annotations/batch")
|
||||
async def bulk_upsert_annotations(project_id: str, body: dict[str, BulkAnnotationItem],
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""COCO/YOLO import: upsert up to 500 images in one commit."""
|
||||
items = {image_id: AnnotationUpsert(**item.model_dump()) for image_id, item in body.items()}
|
||||
return await annotations_svc.bulk_upsert(project_id, items, db)
|
||||
|
||||
|
||||
@router.post("/{project_id}/classes", status_code=201)
|
||||
async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.add_label_class(project_id, body, db)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
|
@ -22,6 +22,28 @@ async def metrics():
|
|||
return get_snapshot()
|
||||
|
||||
|
||||
_usage_cache: dict = {"at": 0.0, "data": None}
|
||||
|
||||
|
||||
@router.get("/storage/usage")
|
||||
async def storage_usage():
|
||||
"""Real bucket bytes/objects for the sidebar meter (10-min cache)."""
|
||||
import time
|
||||
from shared import r2
|
||||
from src.core.config import get_settings
|
||||
now = time.time()
|
||||
if _usage_cache["data"] is not None and now - _usage_cache["at"] < 600:
|
||||
return _usage_cache["data"]
|
||||
try:
|
||||
usage = r2.bucket_usage()
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
data = {"bytes": usage["bytes"], "objects": usage["objects"],
|
||||
"quota_bytes": get_settings().storage_quota_bytes}
|
||||
_usage_cache.update({"at": now, "data": data})
|
||||
return data
|
||||
|
||||
|
||||
@router.post("/admin/backfill-thumbnails")
|
||||
async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)):
|
||||
"""Mint thumbs for images uploaded before thumbnailing existed. Dev-only."""
|
||||
|
|
|
|||
102
src/api/v1/uploads.py
Normal file
102
src/api/v1/uploads.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
"""Resumable multipart uploads: big single files (GB zip/TIFF/video).
|
||||
|
||||
Flow: POST /multipart -> {key, uploadId} -> PUT each part to its presigned
|
||||
URL (parallel, retried per part, resumable via stored ETags) ->
|
||||
POST /multipart/complete. Use for anything over ~100MB; small files keep
|
||||
the single-PUT presign path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Project
|
||||
from shared import r2
|
||||
from src.core.db import get_session
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/uploads", tags=["uploads"])
|
||||
|
||||
PART_SIZE = 8 * 1024 * 1024
|
||||
|
||||
|
||||
class MultipartStart(BaseModel):
|
||||
fileName: str
|
||||
contentType: str = "application/octet-stream"
|
||||
|
||||
|
||||
class MultipartParts(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
numbers: list[int] = Field(max_length=1000)
|
||||
|
||||
|
||||
class PartETag(BaseModel):
|
||||
ETag: str
|
||||
PartNumber: int
|
||||
|
||||
|
||||
class MultipartComplete(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
parts: list[PartETag]
|
||||
|
||||
|
||||
class MultipartAbort(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
|
||||
|
||||
async def _project_slug(project_id: str, db: AsyncSession) -> str:
|
||||
p = await db.get(Project, project_id)
|
||||
if p is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
return p.slug
|
||||
|
||||
|
||||
@router.post("/multipart")
|
||||
async def start_multipart(project_id: str, body: MultipartStart,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
import uuid as _uuid
|
||||
slug = await _project_slug(project_id, db)
|
||||
key = f"uploads/{slug}/{_uuid.uuid4().hex[:12]}-{body.fileName.replace('/', '_')}"
|
||||
try:
|
||||
upload_id = r2.create_multipart(key, body.contentType)
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
|
||||
|
||||
|
||||
@router.post("/multipart/parts")
|
||||
async def part_urls(project_id: str, body: MultipartParts,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
if any(n < 1 or n > 10000 for n in body.numbers):
|
||||
raise HTTPException(400, "invalid part numbers")
|
||||
try:
|
||||
urls = {n: r2.presign_part(body.key, body.uploadId, n) for n in body.numbers}
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
return {"urls": urls}
|
||||
|
||||
|
||||
@router.post("/multipart/complete")
|
||||
async def complete_multipart(project_id: str, body: MultipartComplete,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
try:
|
||||
etag = r2.complete_multipart(body.key, body.uploadId,
|
||||
[p.model_dump() for p in body.parts])
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
|
||||
if not r2.head_exists(body.key):
|
||||
raise HTTPException(400, "object not found after complete")
|
||||
return {"key": body.key, "etag": etag}
|
||||
|
||||
|
||||
@router.post("/multipart/abort")
|
||||
async def abort_multipart(project_id: str, body: MultipartAbort,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
r2.abort_multipart(body.key, body.uploadId)
|
||||
return {"aborted": True}
|
||||
Loading…
Add table
Add a link
Reference in a new issue