feat: add resumable dataset uploads
This commit is contained in:
parent
f8ed08b854
commit
8d40cff227
23 changed files with 913 additions and 21 deletions
|
|
@ -71,3 +71,11 @@ async def invite(body: InviteIn, db: AsyncSession = Depends(get_session)):
|
|||
async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)):
|
||||
from src.services import users as svc
|
||||
return await svc.patch_user(user_id, body.model_dump(), db)
|
||||
|
||||
|
||||
@users_router.delete("/{user_id}")
|
||||
async def delete_user(user_id: str, db: AsyncSession = Depends(get_session),
|
||||
user = Depends(get_current_user)):
|
||||
from src.services import users as svc
|
||||
actor = user.id if user is not None else None
|
||||
return await svc.delete_user(user_id, actor, db)
|
||||
|
|
|
|||
|
|
@ -3,13 +3,21 @@ from __future__ import annotations
|
|||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from shared.schemas import ImageConfirm, PresignRequest
|
||||
from shared.schemas import BatchPresignItem, ImageConfirm, PresignRequest
|
||||
from src.core.db import get_session
|
||||
from src.services import images as svc
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"])
|
||||
|
||||
|
||||
@router.get("/by-id/{image_id}")
|
||||
async def locate_image(project_id: str, image_id: str,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Deep-link support (?imageId=): offset of one image in list order."""
|
||||
from src.services import images as svc
|
||||
return await svc.locate_image(project_id, image_id, db)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_images(project_id: str, status: str | None = Query(default=None),
|
||||
search: str | None = Query(default=None),
|
||||
|
|
@ -26,6 +34,13 @@ async def presign_upload(project_id: str, body: PresignRequest):
|
|||
return svc.build_presign(project_id, body.fileName, body.contentType)
|
||||
|
||||
|
||||
@router.post("/batch-presign")
|
||||
async def batch_presign_upload(project_id: str, body: list[BatchPresignItem],
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""One call for a whole upload batch (≤200 files)."""
|
||||
return await svc.batch_presign(project_id, [b.model_dump() for b in body], db)
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.confirm_image(project_id, body, db)
|
||||
|
|
|
|||
42
src/api/v1/imports.py
Normal file
42
src/api/v1/imports.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.core.db import get_session
|
||||
from src.services import imports as svc
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/imports", tags=["imports"])
|
||||
|
||||
|
||||
class LocalZipIn(BaseModel):
|
||||
path: str
|
||||
|
||||
|
||||
class R2ZipIn(BaseModel):
|
||||
r2_key: str
|
||||
|
||||
|
||||
@router.post("/local-zip", status_code=202)
|
||||
async def start_local_zip(project_id: str, body: LocalZipIn,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Queue a server-side zip import (GB-scale, streamed). Poll the job id."""
|
||||
return await svc.start_local_zip(project_id, body.path, db)
|
||||
|
||||
|
||||
@router.post("/r2-zip", status_code=202)
|
||||
async def start_r2_zip(project_id: str, body: R2ZipIn,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Queue import of a zip already uploaded to R2 (e.g. via multipart)."""
|
||||
return await svc.start_r2_zip(project_id, body.r2_key, db)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_imports(project_id: str, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.list_jobs(project_id, db)
|
||||
|
||||
|
||||
@router.get("/{job_id}")
|
||||
async def get_import(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.get_job(project_id, job_id, db)
|
||||
|
|
@ -3,8 +3,9 @@ from __future__ import annotations
|
|||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from shared.schemas import LabelClassIn, ProjectCreate
|
||||
from shared.schemas import AnnotationUpsert, BulkAnnotationItem, LabelClassIn, ProjectCreate
|
||||
from src.core.db import get_session
|
||||
from src.services import annotations as annotations_svc
|
||||
from src.services import projects as svc
|
||||
|
||||
router = APIRouter(prefix="/projects", tags=["projects"])
|
||||
|
|
@ -35,6 +36,14 @@ async def delete_project(project_id: str, db: AsyncSession = Depends(get_session
|
|||
return await svc.delete_project(project_id, db)
|
||||
|
||||
|
||||
@router.put("/{project_id}/annotations/batch")
|
||||
async def bulk_upsert_annotations(project_id: str, body: dict[str, BulkAnnotationItem],
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""COCO/YOLO import: upsert up to 500 images in one commit."""
|
||||
items = {image_id: AnnotationUpsert(**item.model_dump()) for image_id, item in body.items()}
|
||||
return await annotations_svc.bulk_upsert(project_id, items, db)
|
||||
|
||||
|
||||
@router.post("/{project_id}/classes", status_code=201)
|
||||
async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.add_label_class(project_id, body, db)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
|
@ -22,6 +22,28 @@ async def metrics():
|
|||
return get_snapshot()
|
||||
|
||||
|
||||
_usage_cache: dict = {"at": 0.0, "data": None}
|
||||
|
||||
|
||||
@router.get("/storage/usage")
|
||||
async def storage_usage():
|
||||
"""Real bucket bytes/objects for the sidebar meter (10-min cache)."""
|
||||
import time
|
||||
from shared import r2
|
||||
from src.core.config import get_settings
|
||||
now = time.time()
|
||||
if _usage_cache["data"] is not None and now - _usage_cache["at"] < 600:
|
||||
return _usage_cache["data"]
|
||||
try:
|
||||
usage = r2.bucket_usage()
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
data = {"bytes": usage["bytes"], "objects": usage["objects"],
|
||||
"quota_bytes": get_settings().storage_quota_bytes}
|
||||
_usage_cache.update({"at": now, "data": data})
|
||||
return data
|
||||
|
||||
|
||||
@router.post("/admin/backfill-thumbnails")
|
||||
async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)):
|
||||
"""Mint thumbs for images uploaded before thumbnailing existed. Dev-only."""
|
||||
|
|
|
|||
102
src/api/v1/uploads.py
Normal file
102
src/api/v1/uploads.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
"""Resumable multipart uploads: big single files (GB zip/TIFF/video).
|
||||
|
||||
Flow: POST /multipart -> {key, uploadId} -> PUT each part to its presigned
|
||||
URL (parallel, retried per part, resumable via stored ETags) ->
|
||||
POST /multipart/complete. Use for anything over ~100MB; small files keep
|
||||
the single-PUT presign path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Project
|
||||
from shared import r2
|
||||
from src.core.db import get_session
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/uploads", tags=["uploads"])
|
||||
|
||||
PART_SIZE = 8 * 1024 * 1024
|
||||
|
||||
|
||||
class MultipartStart(BaseModel):
|
||||
fileName: str
|
||||
contentType: str = "application/octet-stream"
|
||||
|
||||
|
||||
class MultipartParts(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
numbers: list[int] = Field(max_length=1000)
|
||||
|
||||
|
||||
class PartETag(BaseModel):
|
||||
ETag: str
|
||||
PartNumber: int
|
||||
|
||||
|
||||
class MultipartComplete(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
parts: list[PartETag]
|
||||
|
||||
|
||||
class MultipartAbort(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
|
||||
|
||||
async def _project_slug(project_id: str, db: AsyncSession) -> str:
|
||||
p = await db.get(Project, project_id)
|
||||
if p is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
return p.slug
|
||||
|
||||
|
||||
@router.post("/multipart")
|
||||
async def start_multipart(project_id: str, body: MultipartStart,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
import uuid as _uuid
|
||||
slug = await _project_slug(project_id, db)
|
||||
key = f"uploads/{slug}/{_uuid.uuid4().hex[:12]}-{body.fileName.replace('/', '_')}"
|
||||
try:
|
||||
upload_id = r2.create_multipart(key, body.contentType)
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
|
||||
|
||||
|
||||
@router.post("/multipart/parts")
|
||||
async def part_urls(project_id: str, body: MultipartParts,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
if any(n < 1 or n > 10000 for n in body.numbers):
|
||||
raise HTTPException(400, "invalid part numbers")
|
||||
try:
|
||||
urls = {n: r2.presign_part(body.key, body.uploadId, n) for n in body.numbers}
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
return {"urls": urls}
|
||||
|
||||
|
||||
@router.post("/multipart/complete")
|
||||
async def complete_multipart(project_id: str, body: MultipartComplete,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
try:
|
||||
etag = r2.complete_multipart(body.key, body.uploadId,
|
||||
[p.model_dump() for p in body.parts])
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
|
||||
if not r2.head_exists(body.key):
|
||||
raise HTTPException(400, "object not found after complete")
|
||||
return {"key": body.key, "etag": etag}
|
||||
|
||||
|
||||
@router.post("/multipart/abort")
|
||||
async def abort_multipart(project_id: str, body: MultipartAbort,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
r2.abort_multipart(body.key, body.uploadId)
|
||||
return {"aborted": True}
|
||||
|
|
@ -46,6 +46,7 @@ class Settings:
|
|||
layout = storage.get("layout", {})
|
||||
presigned = storage.get("presigned_ttl", {})
|
||||
self.r2_layout: dict = layout
|
||||
self.storage_quota_bytes: int = int(storage.get("quota_bytes", 10 * 1024**3))
|
||||
thumb = storage.get("thumbnail", {})
|
||||
self.thumbnail_enabled: bool = bool(thumb.get("enabled", True))
|
||||
self.thumbnail_max_side: int = int(thumb.get("max_side", 512))
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from shared import events
|
|||
from shared.logger import get_logger
|
||||
from shared.tracing import TraceMiddleware
|
||||
from shared.traffic import TrafficMiddleware
|
||||
from src.api.v1 import activities, annotations, auth, events as events_api, images, projects, system
|
||||
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads
|
||||
from src.api.v1.auth import users_router
|
||||
from src.core.config import get_settings
|
||||
from src.core.security import get_current_user
|
||||
|
|
@ -61,6 +61,8 @@ app.include_router(activities.router, prefix=prefix, dependencies=authed)
|
|||
app.include_router(projects.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(images.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(imports.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(uploads.router, prefix=prefix, dependencies=authed)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import time
|
|||
from fastapi import HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Annotation, Image
|
||||
from models import Annotation, Image, Project
|
||||
from shared import events
|
||||
from shared.schemas import AnnotationUpsert
|
||||
|
||||
|
|
@ -29,10 +29,8 @@ async def get_annotation(project_id: str, image_id: str, db: AsyncSession) -> di
|
|||
"flags": ann.flags, "shapes": ann.shapes}
|
||||
|
||||
|
||||
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
|
||||
db: AsyncSession) -> dict:
|
||||
im = await _require_image(project_id, image_id, db)
|
||||
t0 = time.perf_counter()
|
||||
async def _apply_upsert(db: AsyncSession, project_id: str, image_id: str,
|
||||
body: AnnotationUpsert) -> int:
|
||||
ann = await db.get(Annotation, image_id)
|
||||
payload = {"project_id": project_id,
|
||||
"classification_label_id": body.classificationLabelId,
|
||||
|
|
@ -49,6 +47,43 @@ async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpse
|
|||
im.object_count = len(body.shapes)
|
||||
im.classification_label_id = body.classificationLabelId
|
||||
im.flags = body.flags.model_dump()
|
||||
return len(body.shapes)
|
||||
|
||||
|
||||
async def bulk_upsert(project_id: str, items: dict[str, AnnotationUpsert],
|
||||
db: AsyncSession) -> dict:
|
||||
"""COCO/YOLO import path: one commit for the whole batch (≤500 images).
|
||||
|
||||
Imported labels are machine data awaiting human review, so saved rows go
|
||||
to pending_review — that is what the review queue and stats count on.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
if await db.get(Project, project_id) is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
from shared import events
|
||||
import time
|
||||
saved, missing = 0, []
|
||||
t0 = time.perf_counter()
|
||||
for image_id, body in list(items.items())[:500]:
|
||||
im = await db.get(Image, image_id)
|
||||
if im is None or im.project_id != project_id:
|
||||
missing.append(image_id)
|
||||
continue
|
||||
await _apply_upsert(db, project_id, image_id, body)
|
||||
im.status = "pending_review"
|
||||
saved += 1
|
||||
await db.commit()
|
||||
await events.emit("dataset.import", project_id=project_id,
|
||||
payload={"saved": saved, "missing": len(missing)},
|
||||
latency_ms=(time.perf_counter() - t0) * 1000)
|
||||
return {"saved": saved, "missing": missing}
|
||||
|
||||
|
||||
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
|
||||
db: AsyncSession) -> dict:
|
||||
await _require_image(project_id, image_id, db)
|
||||
t0 = time.perf_counter()
|
||||
await _apply_upsert(db, project_id, image_id, body)
|
||||
await db.commit()
|
||||
await events.emit("annotation.save", project_id=project_id, image_id=image_id,
|
||||
payload={"shapeCount": len(body.shapes),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from fastapi import HTTPException
|
|||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Image
|
||||
from models import Image, Project
|
||||
from shared import events, keys, r2
|
||||
from shared.schemas import ImageConfirm
|
||||
|
||||
|
|
@ -40,7 +40,10 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession,
|
|||
q = q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
|
||||
count_q = count_q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
|
||||
total = (await db.execute(count_q)).scalar() or 0
|
||||
rows = (await db.execute(q.order_by(Image.created_at.desc()).limit(limit).offset(offset))).scalars().all()
|
||||
# Deterministic order (id tiebreak): pages must not shift underfoot.
|
||||
rows = (await db.execute(
|
||||
q.order_by(Image.created_at.desc(), Image.id.desc()).limit(limit).offset(offset)
|
||||
)).scalars().all()
|
||||
users = (await db.execute(select(User))).scalars().all()
|
||||
names = {u.id: u.name for u in users}
|
||||
items = [{"id": im.id, "projectId": im.project_id, "fileName": im.file_name,
|
||||
|
|
@ -57,12 +60,44 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession,
|
|||
return {"items": items, "total": total}
|
||||
|
||||
|
||||
async def locate_image(project_id: str, image_id: str, db: AsyncSession) -> dict:
|
||||
"""Offset (0-based) of one image in list order, for ?imageId= deep links."""
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import func, text
|
||||
total = (await db.execute(
|
||||
select(func.count()).where(Image.project_id == project_id))).scalar() or 0
|
||||
row = (await db.execute(text(
|
||||
"SELECT rn FROM (SELECT id, ROW_NUMBER() OVER "
|
||||
"(ORDER BY created_at DESC, id DESC) AS rn FROM images "
|
||||
"WHERE project_id = :pid) t WHERE id = :iid"),
|
||||
{"pid": project_id, "iid": image_id})).first()
|
||||
if row is None:
|
||||
raise HTTPException(404, "image not found")
|
||||
return {"offset": int(row[0]) - 1, "total": total}
|
||||
|
||||
|
||||
def build_presign(project_id: str, file_name: str, content_type: str) -> dict:
|
||||
"""Step 1/2 direct upload: frontend PUTs bytes straight to R2 with this URL."""
|
||||
key = keys.image_key(project_id, "tmp", file_name) # real id assigned on confirm
|
||||
import uuid as _uuid
|
||||
# Unique tmp id per file: duplicate filenames in one batch must not collide.
|
||||
key = keys.image_key(project_id, f"tmp-{_uuid.uuid4().hex[:8]}", file_name)
|
||||
return {"putUrl": r2.presign_put(key, content_type), "r2Key": key}
|
||||
|
||||
|
||||
async def batch_presign(project_id: str, files: list[dict], db: AsyncSession) -> list[dict]:
|
||||
"""One call for a whole batch (≤200): avoids a presign roundtrip per file."""
|
||||
if await db.get(Project, project_id) is None:
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(404, "project not found")
|
||||
out = []
|
||||
for f in files[:200]:
|
||||
slot = build_presign(project_id, f.get("fileName", "upload.jpg"),
|
||||
f.get("contentType", "image/jpeg"))
|
||||
slot["fileName"] = f.get("fileName", "upload.jpg")
|
||||
out.append(slot)
|
||||
return out
|
||||
|
||||
|
||||
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession) -> dict:
|
||||
"""Step 2/2: metadata row after the R2 PUT succeeds."""
|
||||
if not r2.head_exists(body.r2Key):
|
||||
|
|
|
|||
289
src/services/imports.py
Normal file
289
src/services/imports.py
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
"""Server-side dataset import: stream a big zip (GBs) without loading it to RAM.
|
||||
|
||||
One image in memory at a time for reads; network puts run concurrently;
|
||||
DB commits in batches of 100. Re-runnable: files already in the project
|
||||
(by file_name) are skipped.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import zipfile
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Image, ImportJob, LabelClass, Project
|
||||
from shared import events, keys, r2
|
||||
from shared.logger import get_logger
|
||||
from shared.schemas import AnnotationShape, AnnotationUpsert, Flags
|
||||
from shared.thumbnails import IMMUTABLE, resize_to_thumbnail
|
||||
from src.core.db import get_session_factory
|
||||
|
||||
log = get_logger("danable.imports")
|
||||
|
||||
IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp", ".bmp")
|
||||
PUT_CONCURRENCY = 24
|
||||
DB_BATCH = 100
|
||||
|
||||
|
||||
def _basename(name: str) -> str:
|
||||
return name.split("/")[-1]
|
||||
|
||||
|
||||
def _is_image(name: str) -> bool:
|
||||
return name.lower().endswith(IMAGE_EXTS)
|
||||
|
||||
|
||||
def build_bulk_from_coco(coco: dict, file_to_image: dict[str, str],
|
||||
cat_to_class: dict[str, dict]) -> dict[str, AnnotationUpsert]:
|
||||
"""Pure mapping: COCO annotations -> bulk-upsert payload (unit-tested)."""
|
||||
cat_name = {str(c.get("id")): c.get("name", f"class_{c.get('id')}")
|
||||
for c in coco.get("categories", [])}
|
||||
by_image: dict[str, list] = {}
|
||||
for ann in coco.get("annotations", []):
|
||||
meta = next((i for i in coco.get("images", [])
|
||||
if str(i.get("id")) == str(ann.get("image_id"))), None)
|
||||
if meta is None:
|
||||
continue
|
||||
fname = _basename(meta.get("file_name", ""))
|
||||
image_id = file_to_image.get(fname)
|
||||
if image_id is None:
|
||||
continue
|
||||
label = cat_name.get(str(ann.get("category_id")), "unknown")
|
||||
cls = cat_to_class.get(label.lower())
|
||||
if cls is None:
|
||||
continue
|
||||
shape = {"labelId": cls["id"], "labelName": cls["name"], "color": cls["color"]}
|
||||
seg = ann.get("segmentation")
|
||||
if isinstance(seg, list) and seg and isinstance(seg[0], list) and len(seg[0]) >= 6:
|
||||
ring = seg[0]
|
||||
pts = [{"x": int(ring[i]), "y": int(ring[i + 1])}
|
||||
for i in range(0, len(ring) - 1, 2)]
|
||||
shape.update({"type": "polygon", "points": pts})
|
||||
elif isinstance(ann.get("bbox"), list) and len(ann["bbox"]) == 4:
|
||||
x, y, w, h = (int(v) for v in ann["bbox"])
|
||||
shape.update({"type": "bbox", "bbox": {"x": x, "y": y, "width": w, "height": h}})
|
||||
else:
|
||||
continue # RLE/crowd without bbox: skipped
|
||||
by_image.setdefault(image_id, []).append(shape)
|
||||
payload: dict[str, AnnotationUpsert] = {}
|
||||
for image_id, shapes in by_image.items():
|
||||
payload[image_id] = AnnotationUpsert(
|
||||
flags=Flags(),
|
||||
shapes=[AnnotationShape(id=f"zip-{k}", index=k + 1, visible=True, **s)
|
||||
for k, s in enumerate(shapes)],
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _job_to_dict(job: ImportJob) -> dict:
|
||||
return {"id": job.id, "project_id": job.project_id, "kind": job.kind,
|
||||
"status": job.status, "source": job.source,
|
||||
"total_files": job.total_files, "done_files": job.done_files,
|
||||
"saved_annotations": job.saved_annotations, "error": job.error}
|
||||
|
||||
|
||||
async def list_jobs(project_id: str, db: AsyncSession) -> list[dict]:
|
||||
rows = (await db.execute(select(ImportJob).where(
|
||||
ImportJob.project_id == project_id).order_by(ImportJob.created_at.desc()))
|
||||
).scalars().all()
|
||||
return [_job_to_dict(j) for j in rows]
|
||||
|
||||
|
||||
async def get_job(project_id: str, job_id: str, db: AsyncSession) -> dict:
|
||||
job = await db.get(ImportJob, job_id)
|
||||
if job is None or job.project_id != project_id:
|
||||
raise HTTPException(404, "import job not found")
|
||||
return _job_to_dict(job)
|
||||
|
||||
|
||||
async def start_local_zip(project_id: str, path: str, db: AsyncSession) -> dict:
|
||||
p = await db.get(Project, project_id)
|
||||
if p is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
if not os.path.isabs(path) or not path.endswith(".zip") or not os.path.exists(path):
|
||||
raise HTTPException(400, "path must be an existing absolute .zip file")
|
||||
job = ImportJob(project_id=project_id, kind="zip-coco", status="queued", source=path)
|
||||
db.add(job)
|
||||
await db.commit()
|
||||
await db.refresh(job)
|
||||
asyncio.create_task(_run_zip_coco(job.id, project_id, path))
|
||||
return _job_to_dict(job)
|
||||
|
||||
|
||||
async def start_r2_zip(project_id: str, r2_key: str, db: AsyncSession) -> dict:
|
||||
"""Import a zip already in R2 (e.g. multipart-uploaded from the browser):
|
||||
stream it to local temp (bounded RAM), run the standard pipeline, delete."""
|
||||
p = await db.get(Project, project_id)
|
||||
if p is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
if not r2_key.endswith(".zip"):
|
||||
raise HTTPException(400, "r2_key must point to a .zip object")
|
||||
job = ImportJob(project_id=project_id, kind="zip-coco",
|
||||
status="queued", source=f"r2:{r2_key}")
|
||||
db.add(job)
|
||||
await db.commit()
|
||||
await db.refresh(job)
|
||||
asyncio.create_task(_run_r2_zip(job.id, project_id, r2_key))
|
||||
return _job_to_dict(job)
|
||||
|
||||
|
||||
async def _run_r2_zip(job_id: str, project_id: str, r2_key: str) -> None:
|
||||
import tempfile
|
||||
tmp = tempfile.mktemp(prefix=f"danable-import-{job_id}-", suffix=".zip")
|
||||
try:
|
||||
await asyncio.to_thread(r2.download_to_file, r2_key, tmp)
|
||||
await _run_zip_coco(job_id, project_id, tmp)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _run_zip_coco(job_id: str, project_id: str, path: str) -> None:
|
||||
t_all = time.perf_counter()
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
job = await db.get(ImportJob, job_id)
|
||||
if job is None:
|
||||
return
|
||||
try:
|
||||
job.status = "running"
|
||||
await db.commit()
|
||||
project = await db.get(Project, project_id)
|
||||
if project is None:
|
||||
raise RuntimeError("project gone")
|
||||
|
||||
zf = zipfile.ZipFile(path)
|
||||
names = zf.namelist()
|
||||
coco_name = next((n for n in names if _basename(n) == "annotations.json"),
|
||||
next((n for n in names if n.endswith(".json")), None))
|
||||
if coco_name is None:
|
||||
raise RuntimeError("no COCO json found in zip")
|
||||
coco = json.loads(zf.read(coco_name))
|
||||
|
||||
# 1. ensure label classes exist
|
||||
palette = ["#3B82F6", "#10B981", "#F59E0B", "#A855F7", "#EF4444", "#06B6D4"]
|
||||
existing = (await db.execute(select(LabelClass).where(
|
||||
LabelClass.project_id == project_id))).scalars().all()
|
||||
have = {c.name.lower() for c in existing}
|
||||
for i, c in enumerate(coco.get("categories", [])):
|
||||
if c.get("name", "").lower() not in have:
|
||||
db.add(LabelClass(project_id=project_id, name=c["name"],
|
||||
color=palette[(len(existing) + i) % len(palette)]))
|
||||
have.add(c.get("name", "").lower())
|
||||
await db.commit()
|
||||
|
||||
# 2. stream images: sequential reads, concurrent puts, batched commits
|
||||
present = set((await db.execute(select(Image.file_name).where(
|
||||
Image.project_id == project_id))).scalars().all())
|
||||
entries = [n for n in names if _is_image(n) and _basename(n) not in present]
|
||||
job.total_files = len(entries)
|
||||
await db.commit()
|
||||
log.info("import %s: %d new files of %d", job_id, len(entries), len(names))
|
||||
|
||||
dims = {_basename(i.get("file_name", "")):
|
||||
(i.get("width", 0), i.get("height", 0))
|
||||
for i in coco.get("images", [])}
|
||||
sem = asyncio.Semaphore(PUT_CONCURRENCY)
|
||||
|
||||
async def process_batch(batch: list[tuple[str, bytes]]) -> None:
|
||||
# Concurrent network puts (bounded); then one DB commit.
|
||||
results = await asyncio.gather(*[
|
||||
put_all(
|
||||
keys.image_key(project.slug, f"zip-{uuid.uuid4().hex[:8]}", fname),
|
||||
keys.thumbnail_key(project.slug, fname.rsplit(".", 1)[0]),
|
||||
keys.thumbnail_small_key(project.slug, fname.rsplit(".", 1)[0]),
|
||||
data,
|
||||
)
|
||||
for fname, data in batch
|
||||
])
|
||||
rows = []
|
||||
for (fname, data), keys3 in zip(batch, results):
|
||||
r2_key, thumb_key, small_key = keys3
|
||||
w, h = dims.get(fname, (0, 0))
|
||||
rows.append(Image(
|
||||
project_id=project_id, file_name=fname, r2_key=r2_key,
|
||||
thumbnail_r2_key=thumb_key, thumbnail_small_r2_key=small_key,
|
||||
width=w, height=h, status="unassigned", flags={}))
|
||||
db.add_all(rows)
|
||||
await db.commit()
|
||||
job.done_files = (job.done_files or 0) + len(rows)
|
||||
await db.commit()
|
||||
|
||||
async def put_all(r2_key: str, thumb_key: str, small_key: str,
|
||||
data: bytes) -> tuple[str, str, str]:
|
||||
async with sem:
|
||||
await asyncio.to_thread(r2.put_bytes, r2_key, data, "image/jpeg")
|
||||
try:
|
||||
big = await asyncio.to_thread(resize_to_thumbnail, data, 512, 70)
|
||||
tiny = await asyncio.to_thread(resize_to_thumbnail, data, 128, 65)
|
||||
await asyncio.to_thread(
|
||||
r2.put_bytes, thumb_key, big, "image/jpeg", IMMUTABLE)
|
||||
await asyncio.to_thread(
|
||||
r2.put_bytes, small_key, tiny, "image/jpeg", IMMUTABLE)
|
||||
except Exception as e:
|
||||
log.warning("thumb inline failed: %s", e)
|
||||
return (r2_key, thumb_key, small_key)
|
||||
|
||||
batch: list[tuple[str, str]] = []
|
||||
|
||||
async def flush_names() -> None:
|
||||
if not batch:
|
||||
return
|
||||
# Sequential disk reads (zipfile is not thread-safe), then
|
||||
# concurrent network puts for the whole batch at once.
|
||||
loaded = [(fname, zf.read(name)) for name, fname in batch]
|
||||
await process_batch(loaded)
|
||||
batch.clear()
|
||||
|
||||
for name in entries:
|
||||
batch.append((name, _basename(name)))
|
||||
if len(batch) >= 48:
|
||||
await flush_names()
|
||||
await flush_names()
|
||||
|
||||
# 3. annotations via the bulk path in 500-image chunks
|
||||
rows = (await db.execute(select(Image.id, Image.file_name).where(
|
||||
Image.project_id == project_id))).all()
|
||||
file_to_image = {fname: iid for iid, fname in rows}
|
||||
classes = (await db.execute(select(LabelClass).where(
|
||||
LabelClass.project_id == project_id))).scalars().all()
|
||||
cat_to_class = {c.name.lower(): {"id": c.id, "name": c.name, "color": c.color}
|
||||
for c in classes}
|
||||
from src.services import annotations as ann_svc
|
||||
payload = build_bulk_from_coco(coco, file_to_image, cat_to_class)
|
||||
saved = 0
|
||||
chunk: dict[str, AnnotationUpsert] = {}
|
||||
for iid, body in payload.items():
|
||||
chunk[iid] = body
|
||||
if len(chunk) >= 500:
|
||||
out = await ann_svc.bulk_upsert(project_id, chunk, db)
|
||||
saved += out["saved"]
|
||||
chunk = {}
|
||||
if chunk:
|
||||
out = await ann_svc.bulk_upsert(project_id, chunk, db)
|
||||
saved += out["saved"]
|
||||
job.saved_annotations = saved
|
||||
job.status = "done"
|
||||
await db.commit()
|
||||
await events.emit("dataset.import", project_id=project_id,
|
||||
payload={"via": "zip", "files": job.done_files, "saved": saved},
|
||||
latency_ms=(time.perf_counter() - t_all) * 1000)
|
||||
log.info("import %s done: %d files, %d annotations", job_id, job.done_files, saved)
|
||||
except Exception as e:
|
||||
log.warning("import %s failed: %s", job_id, e)
|
||||
try:
|
||||
job = await db.get(ImportJob, job_id)
|
||||
if job is not None:
|
||||
job.status = "error"
|
||||
job.error = str(e)[:2000]
|
||||
await db.commit()
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -44,16 +44,23 @@ async def create_project(body: ProjectCreate, db: AsyncSession) -> dict:
|
|||
|
||||
|
||||
async def _stats(project_id: str, db: AsyncSession) -> dict:
|
||||
counts: dict[str, int] = {}
|
||||
for st in ("unassigned", "in_progress", "pending_review", "completed"):
|
||||
counts[st] = (await db.execute(
|
||||
select(func.count()).where(Image.project_id == project_id, Image.status == st))).scalar() or 0
|
||||
"""One GROUP BY instead of five counts + a sum: the list endpoint calls
|
||||
this per project, so roundtrips dominate at 10+ projects."""
|
||||
rows = (await db.execute(
|
||||
select(Image.status, func.count(),
|
||||
func.coalesce(func.sum(Image.object_count), 0))
|
||||
.where(Image.project_id == project_id)
|
||||
.group_by(Image.status))).all()
|
||||
counts = {st: 0 for st in ("unassigned", "in_progress", "pending_review", "completed")}
|
||||
objects = 0
|
||||
for st, n, o in rows:
|
||||
if st in counts:
|
||||
counts[st] = n or 0
|
||||
objects += int(o or 0)
|
||||
total = sum(counts.values())
|
||||
objects = (await db.execute(
|
||||
select(func.coalesce(func.sum(Image.object_count), 0)).where(Image.project_id == project_id))).scalar() or 0
|
||||
return {"totalImages": total, "unassigned": counts["unassigned"], "inProgress": counts["in_progress"],
|
||||
"pendingReview": counts["pending_review"], "completed": counts["completed"],
|
||||
"totalObjects": int(objects)}
|
||||
"totalObjects": objects}
|
||||
|
||||
|
||||
async def _members(project_id: str, db: AsyncSession) -> list[dict]:
|
||||
|
|
@ -94,13 +101,28 @@ async def _thumbnail_url(project_id: str, db: AsyncSession) -> str:
|
|||
return ""
|
||||
|
||||
|
||||
async def _class_counts(project_id: str, db: AsyncSession) -> dict[str, int]:
|
||||
"""Per-label object totals straight from stored shapes (always exact,
|
||||
including bulk imports and deletions — no counter drift possible)."""
|
||||
from sqlalchemy import text
|
||||
rows = (await db.execute(text(
|
||||
"SELECT elem->>'labelId' AS lid, COUNT(*) FROM annotations, "
|
||||
"LATERAL json_array_elements(CASE WHEN shapes IS NULL THEN '[]' "
|
||||
"WHEN json_typeof(shapes) = 'array' THEN shapes ELSE '[]' END) AS elem "
|
||||
"WHERE project_id = :pid GROUP BY lid"),
|
||||
{"pid": project_id})).all()
|
||||
return {lid: n for lid, n in rows if lid}
|
||||
|
||||
|
||||
async def _project_card(p: Project, db: AsyncSession) -> dict:
|
||||
classes = (await db.execute(select(LabelClass).where(LabelClass.project_id == p.id))).scalars().all()
|
||||
counts = await _class_counts(p.id, db)
|
||||
return {"id": p.id, "slug": p.slug, "name": p.name, "taskType": p.task_type,
|
||||
"status": p.status, "description": p.description,
|
||||
"thumbnailUrl": await _thumbnail_url(p.id, db),
|
||||
"updatedAt": p.updated_at.isoformat() if p.updated_at else "",
|
||||
"classes": [{"id": c.id, "name": c.name, "color": c.color} for c in classes],
|
||||
"classes": [{"id": c.id, "name": c.name, "color": c.color,
|
||||
"count": counts.get(c.id, 0)} for c in classes],
|
||||
"stats": await _stats(p.id, db),
|
||||
"members": await _members(p.id, db)}
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,17 @@ async def invite_user(email: str, role: str, project_ids: list[str], db: AsyncSe
|
|||
return to_public(u)
|
||||
|
||||
|
||||
async def delete_user(user_id: str, actor_id: str | None, db: AsyncSession) -> dict:
|
||||
u = await db.get(User, user_id)
|
||||
if u is None:
|
||||
raise HTTPException(404, "user not found")
|
||||
if actor_id is not None and u.id == actor_id:
|
||||
raise HTTPException(400, "cannot delete your own account")
|
||||
await db.delete(u)
|
||||
await db.commit()
|
||||
return {"deleted": user_id}
|
||||
|
||||
|
||||
async def patch_user(user_id: str, patch: dict, db: AsyncSession) -> dict:
|
||||
u = await db.get(User, user_id)
|
||||
if u is None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue