feat: add resumable dataset uploads

This commit is contained in:
hailt 2026-11-12 17:31:42 +07:00
parent f8ed08b854
commit 8d40cff227
23 changed files with 913 additions and 21 deletions

View file

@ -15,6 +15,8 @@ database:
storage:
bucket_default: danagble
# Reference quota for the sidebar meter (R2 free tier = 10 GiB).
quota_bytes: 10737418240
# R2 key layout. {project} = project slug, {image} = image id.
layout:
image: "datasets/{project}/{image}-{filename}"

View file

@ -15,6 +15,8 @@ database:
storage:
bucket_default: danagble
# Reference quota for the sidebar meter (R2 free tier = 10 GiB).
quota_bytes: 10737418240
# R2 key layout. {project} = project slug, {image} = image id.
layout:
image: "datasets/{project}/{image}-{filename}"

View file

@ -24,6 +24,7 @@ Interactive docs: `http://localhost:8000/docs`.
|---|---|---|
| GET | `/health` | Liveness incl. DB check, `{ ok: true }` |
| GET | `/metrics` | Phase-1 traffic counters per route |
| GET | `/storage/usage` | Real bucket `{ bytes, objects, quota_bytes }` (10-min cache) |
| GET | `/` (root, no prefix) | Name, docs and health links |
| POST | `/admin/init-db` | Create tables, seed dev admin. Dev-only |
| POST | `/admin/backfill-thumbnails?limit=100` | Mint missing thumbs. Dev-only |
@ -97,6 +98,44 @@ GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0
`PATCH /projects/{id}/images/{imageId}?status=completed` — status transition.
`POST /projects/{id}/images/batch-presign` — one call for ≤200 files:
`[{ fileName, contentType }]` → `[{ fileName, putUrl, r2Key }]`
(keys carry unique tmp ids, so duplicate filenames never collide).
`GET /projects/{id}/images/by-id/{imageId}` — `{ offset, total }` in list
order (deterministic `created_at DESC, id DESC`); backs `?imageId=` deep
links into thousand-image datasets.
## Resumable multipart uploads (files ~100MB and up)
Small files keep the single-PUT presign path. Big single files
(GB zips, TIFF, video) go multipart, straight to R2:
1. `POST /projects/{id}/uploads/multipart` `{ fileName, contentType }` →
`{ key, uploadId, partSize: 8388608 }`.
2. `POST .../uploads/multipart/parts` `{ key, uploadId, numbers: [1..N] }`
→ `{ urls: { "1": "...", ... } }` (1h TTL each).
3. PUT every part (parallel, per-part retry; store ETags client-side —
the upload resumes after network drops instead of restarting).
4. `POST .../uploads/multipart/complete` `{ key, uploadId,
parts: [{ ETag, PartNumber }] }` → `{ key, etag }`.
5. `POST .../uploads/multipart/abort` to cancel and free the upload.
ETag response headers are exposed via the bucket CORS policy, which the
frontend needs to read them.
## Server-side dataset imports (GB-scale zips)
`POST /projects/{id}/imports/local-zip` `{ path }` (absolute server path)
or `POST /projects/{id}/imports/r2-zip` `{ r2_key }` (zip already uploaded,
e.g. via multipart) → `202 { id, status: queued, ... }`. The worker streams
the archive (one image in RAM at a time), uploads originals + 512/128px
thumbs concurrently, commits DB rows in batches, then bulk-upserts COCO
annotations. Re-running skips files already present.
`GET /projects/{id}/imports` and `GET /projects/{id}/imports/{jobId}` poll
`{ status, total_files, done_files, saved_annotations, error }`.
## Annotations (auth)
`GET /projects/{pid}/images/{iid}/annotation` — one row per image:
@ -109,6 +148,10 @@ GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0
"visible": true, "bbox": { "x": 10, "y": 20, "width": 100, "height": 80 } }] }
```
`PUT /projects/{id}/annotations/batch` with `{ [imageId]: { classificationLabelId?, flags?, shapes[] } }`
upserts up to 500 images in one commit (COCO/YOLO import path) →
`{ saved, missing: [...] }`.
`PUT` same path with `{ classificationLabelId?, flags?, shapes[] }` upserts
(one row per image, 404 when the image does not exist) and syncs
`object_count` / flags onto the image row. Also emits an `annotation.save`

View file

@ -5,10 +5,11 @@ from models.tables import (
Annotation,
ExportJob,
Image,
ImportJob,
LabelClass,
Project,
User,
)
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
"LabelClass", "Project", "User"]
"ImportJob", "LabelClass", "Project", "User"]

View file

@ -104,6 +104,22 @@ class ActionEvent(Base):
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class ImportJob(Base):
"""Server-side dataset import (e.g. 3GB zip): polled by the UI/admin."""
__tablename__ = "imports"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
project_id: Mapped[str] = mapped_column(String(32), index=True)
kind: Mapped[str] = mapped_column(String(32), default="zip-coco")
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
source: Mapped[str] = mapped_column(String(1024), default="")
total_files: Mapped[int] = mapped_column(Integer, default=0)
done_files: Mapped[int] = mapped_column(Integer, default=0)
saved_annotations: Mapped[int] = mapped_column(Integer, default=0)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class ExportJob(Base):
__tablename__ = "exports"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)

View file

@ -64,6 +64,67 @@ def get_bytes(key: str) -> bytes:
return obj["Body"].read()
def bucket_usage(prefix: str = "") -> dict:
"""Sum bytes + object count via paginated LIST (no download)."""
s = get_settings()
total_bytes, total_objects = 0, 0
paginator = _client().get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=s.r2_bucket, Prefix=prefix):
for obj in page.get("Contents", []):
total_bytes += obj.get("Size", 0)
total_objects += 1
return {"bytes": total_bytes, "objects": total_objects}
def create_multipart(key: str, content_type: str = "application/octet-stream") -> str:
"""Start a multipart upload; returns upload_id. Parts upload in parallel,
each retriable; the upload survives network drops via the stored id."""
s = get_settings()
res = _client().create_multipart_upload(
Bucket=s.r2_bucket, Key=key, ContentType=content_type)
return res["UploadId"]
def presign_part(key: str, upload_id: str, part_number: int,
expires_in: int = 3600) -> str:
s = get_settings()
return _client().generate_presigned_url(
"upload_part",
Params={"Bucket": s.r2_bucket, "Key": key,
"UploadId": upload_id, "PartNumber": part_number},
ExpiresIn=expires_in)
def complete_multipart(key: str, upload_id: str, parts: list[dict]) -> str:
"""parts: [{ETag, PartNumber}]. Returns final ETag."""
s = get_settings()
res = _client().complete_multipart_upload(
Bucket=s.r2_bucket, Key=key, UploadId=upload_id,
MultipartUpload={"Parts": [{"ETag": p["ETag"], "PartNumber": p["PartNumber"]}
for p in sorted(parts, key=lambda x: x["PartNumber"])]})
return res.get("ETag", "")
def abort_multipart(key: str, upload_id: str) -> None:
s = get_settings()
try:
_client().abort_multipart_upload(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
except Exception:
pass
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
s = get_settings()
body = _client().get_object(Bucket=s.r2_bucket, Key=key)["Body"]
total = 0
with open(dest_path, "wb") as f:
for chunk in body.iter_chunks(chunk_size=chunk_bytes):
f.write(chunk)
total += len(chunk)
return total
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
cache_control: str | None = None) -> None:
s = get_settings()

View file

@ -73,3 +73,14 @@ class ImageConfirm(BaseModel):
class PresignRequest(BaseModel):
fileName: str
contentType: str = "image/jpeg"
class BatchPresignItem(BaseModel):
fileName: str
contentType: str = "image/jpeg"
class BulkAnnotationItem(BaseModel):
classificationLabelId: str | None = None
flags: Flags = Field(default_factory=Flags)
shapes: list[AnnotationShape] = Field(default_factory=list)

View file

@ -71,3 +71,11 @@ async def invite(body: InviteIn, db: AsyncSession = Depends(get_session)):
async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)):
from src.services import users as svc
return await svc.patch_user(user_id, body.model_dump(), db)
@users_router.delete("/{user_id}")
async def delete_user(user_id: str, db: AsyncSession = Depends(get_session),
user = Depends(get_current_user)):
from src.services import users as svc
actor = user.id if user is not None else None
return await svc.delete_user(user_id, actor, db)

View file

@ -3,13 +3,21 @@ from __future__ import annotations
from fastapi import APIRouter, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from shared.schemas import ImageConfirm, PresignRequest
from shared.schemas import BatchPresignItem, ImageConfirm, PresignRequest
from src.core.db import get_session
from src.services import images as svc
router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"])
@router.get("/by-id/{image_id}")
async def locate_image(project_id: str, image_id: str,
db: AsyncSession = Depends(get_session)):
"""Deep-link support (?imageId=): offset of one image in list order."""
from src.services import images as svc
return await svc.locate_image(project_id, image_id, db)
@router.get("")
async def list_images(project_id: str, status: str | None = Query(default=None),
search: str | None = Query(default=None),
@ -26,6 +34,13 @@ async def presign_upload(project_id: str, body: PresignRequest):
return svc.build_presign(project_id, body.fileName, body.contentType)
@router.post("/batch-presign")
async def batch_presign_upload(project_id: str, body: list[BatchPresignItem],
db: AsyncSession = Depends(get_session)):
"""One call for a whole upload batch (≤200 files)."""
return await svc.batch_presign(project_id, [b.model_dump() for b in body], db)
@router.post("", status_code=201)
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)):
return await svc.confirm_image(project_id, body, db)

42
src/api/v1/imports.py Normal file
View file

@ -0,0 +1,42 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from src.core.db import get_session
from src.services import imports as svc
router = APIRouter(prefix="/projects/{project_id}/imports", tags=["imports"])
class LocalZipIn(BaseModel):
path: str
class R2ZipIn(BaseModel):
r2_key: str
@router.post("/local-zip", status_code=202)
async def start_local_zip(project_id: str, body: LocalZipIn,
db: AsyncSession = Depends(get_session)):
"""Queue a server-side zip import (GB-scale, streamed). Poll the job id."""
return await svc.start_local_zip(project_id, body.path, db)
@router.post("/r2-zip", status_code=202)
async def start_r2_zip(project_id: str, body: R2ZipIn,
db: AsyncSession = Depends(get_session)):
"""Queue import of a zip already uploaded to R2 (e.g. via multipart)."""
return await svc.start_r2_zip(project_id, body.r2_key, db)
@router.get("")
async def list_imports(project_id: str, db: AsyncSession = Depends(get_session)):
return await svc.list_jobs(project_id, db)
@router.get("/{job_id}")
async def get_import(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
return await svc.get_job(project_id, job_id, db)

View file

@ -3,8 +3,9 @@ from __future__ import annotations
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from shared.schemas import LabelClassIn, ProjectCreate
from shared.schemas import AnnotationUpsert, BulkAnnotationItem, LabelClassIn, ProjectCreate
from src.core.db import get_session
from src.services import annotations as annotations_svc
from src.services import projects as svc
router = APIRouter(prefix="/projects", tags=["projects"])
@ -35,6 +36,14 @@ async def delete_project(project_id: str, db: AsyncSession = Depends(get_session
return await svc.delete_project(project_id, db)
@router.put("/{project_id}/annotations/batch")
async def bulk_upsert_annotations(project_id: str, body: dict[str, BulkAnnotationItem],
db: AsyncSession = Depends(get_session)):
"""COCO/YOLO import: upsert up to 500 images in one commit."""
items = {image_id: AnnotationUpsert(**item.model_dump()) for image_id, item in body.items()}
return await annotations_svc.bulk_upsert(project_id, items, db)
@router.post("/{project_id}/classes", status_code=201)
async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)):
return await svc.add_label_class(project_id, body, db)

View file

@ -1,6 +1,6 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
@ -22,6 +22,28 @@ async def metrics():
return get_snapshot()
_usage_cache: dict = {"at": 0.0, "data": None}
@router.get("/storage/usage")
async def storage_usage():
"""Real bucket bytes/objects for the sidebar meter (10-min cache)."""
import time
from shared import r2
from src.core.config import get_settings
now = time.time()
if _usage_cache["data"] is not None and now - _usage_cache["at"] < 600:
return _usage_cache["data"]
try:
usage = r2.bucket_usage()
except Exception as e:
raise HTTPException(503, f"storage unreachable: {e}")
data = {"bytes": usage["bytes"], "objects": usage["objects"],
"quota_bytes": get_settings().storage_quota_bytes}
_usage_cache.update({"at": now, "data": data})
return data
@router.post("/admin/backfill-thumbnails")
async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)):
"""Mint thumbs for images uploaded before thumbnailing existed. Dev-only."""

102
src/api/v1/uploads.py Normal file
View file

@ -0,0 +1,102 @@
"""Resumable multipart uploads: big single files (GB zip/TIFF/video).
Flow: POST /multipart -> {key, uploadId} -> PUT each part to its presigned
URL (parallel, retried per part, resumable via stored ETags) ->
POST /multipart/complete. Use for anything over ~100MB; small files keep
the single-PUT presign path.
"""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from models import Project
from shared import r2
from src.core.db import get_session
router = APIRouter(prefix="/projects/{project_id}/uploads", tags=["uploads"])
PART_SIZE = 8 * 1024 * 1024
class MultipartStart(BaseModel):
fileName: str
contentType: str = "application/octet-stream"
class MultipartParts(BaseModel):
key: str
uploadId: str
numbers: list[int] = Field(max_length=1000)
class PartETag(BaseModel):
ETag: str
PartNumber: int
class MultipartComplete(BaseModel):
key: str
uploadId: str
parts: list[PartETag]
class MultipartAbort(BaseModel):
key: str
uploadId: str
async def _project_slug(project_id: str, db: AsyncSession) -> str:
p = await db.get(Project, project_id)
if p is None:
raise HTTPException(404, "project not found")
return p.slug
@router.post("/multipart")
async def start_multipart(project_id: str, body: MultipartStart,
db: AsyncSession = Depends(get_session)):
import uuid as _uuid
slug = await _project_slug(project_id, db)
key = f"uploads/{slug}/{_uuid.uuid4().hex[:12]}-{body.fileName.replace('/', '_')}"
try:
upload_id = r2.create_multipart(key, body.contentType)
except Exception as e:
raise HTTPException(503, f"storage unreachable: {e}")
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
@router.post("/multipart/parts")
async def part_urls(project_id: str, body: MultipartParts,
db: AsyncSession = Depends(get_session)):
await _project_slug(project_id, db)
if any(n < 1 or n > 10000 for n in body.numbers):
raise HTTPException(400, "invalid part numbers")
try:
urls = {n: r2.presign_part(body.key, body.uploadId, n) for n in body.numbers}
except Exception as e:
raise HTTPException(503, f"storage unreachable: {e}")
return {"urls": urls}
@router.post("/multipart/complete")
async def complete_multipart(project_id: str, body: MultipartComplete,
db: AsyncSession = Depends(get_session)):
await _project_slug(project_id, db)
try:
etag = r2.complete_multipart(body.key, body.uploadId,
[p.model_dump() for p in body.parts])
except Exception as e:
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
if not r2.head_exists(body.key):
raise HTTPException(400, "object not found after complete")
return {"key": body.key, "etag": etag}
@router.post("/multipart/abort")
async def abort_multipart(project_id: str, body: MultipartAbort,
db: AsyncSession = Depends(get_session)):
await _project_slug(project_id, db)
r2.abort_multipart(body.key, body.uploadId)
return {"aborted": True}

View file

@ -46,6 +46,7 @@ class Settings:
layout = storage.get("layout", {})
presigned = storage.get("presigned_ttl", {})
self.r2_layout: dict = layout
self.storage_quota_bytes: int = int(storage.get("quota_bytes", 10 * 1024**3))
thumb = storage.get("thumbnail", {})
self.thumbnail_enabled: bool = bool(thumb.get("enabled", True))
self.thumbnail_max_side: int = int(thumb.get("max_side", 512))

View file

@ -11,7 +11,7 @@ from shared import events
from shared.logger import get_logger
from shared.tracing import TraceMiddleware
from shared.traffic import TrafficMiddleware
from src.api.v1 import activities, annotations, auth, events as events_api, images, projects, system
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads
from src.api.v1.auth import users_router
from src.core.config import get_settings
from src.core.security import get_current_user
@ -61,6 +61,8 @@ app.include_router(activities.router, prefix=prefix, dependencies=authed)
app.include_router(projects.router, prefix=prefix, dependencies=authed)
app.include_router(images.router, prefix=prefix, dependencies=authed)
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
app.include_router(imports.router, prefix=prefix, dependencies=authed)
app.include_router(uploads.router, prefix=prefix, dependencies=authed)
@app.get("/")

View file

@ -6,7 +6,7 @@ import time
from fastapi import HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from models import Annotation, Image
from models import Annotation, Image, Project
from shared import events
from shared.schemas import AnnotationUpsert
@ -29,10 +29,8 @@ async def get_annotation(project_id: str, image_id: str, db: AsyncSession) -> di
"flags": ann.flags, "shapes": ann.shapes}
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
db: AsyncSession) -> dict:
im = await _require_image(project_id, image_id, db)
t0 = time.perf_counter()
async def _apply_upsert(db: AsyncSession, project_id: str, image_id: str,
body: AnnotationUpsert) -> int:
ann = await db.get(Annotation, image_id)
payload = {"project_id": project_id,
"classification_label_id": body.classificationLabelId,
@ -49,6 +47,43 @@ async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpse
im.object_count = len(body.shapes)
im.classification_label_id = body.classificationLabelId
im.flags = body.flags.model_dump()
return len(body.shapes)
async def bulk_upsert(project_id: str, items: dict[str, AnnotationUpsert],
db: AsyncSession) -> dict:
"""COCO/YOLO import path: one commit for the whole batch (≤500 images).
Imported labels are machine data awaiting human review, so saved rows go
to pending_review — that is what the review queue and stats count on.
"""
from fastapi import HTTPException
if await db.get(Project, project_id) is None:
raise HTTPException(404, "project not found")
from shared import events
import time
saved, missing = 0, []
t0 = time.perf_counter()
for image_id, body in list(items.items())[:500]:
im = await db.get(Image, image_id)
if im is None or im.project_id != project_id:
missing.append(image_id)
continue
await _apply_upsert(db, project_id, image_id, body)
im.status = "pending_review"
saved += 1
await db.commit()
await events.emit("dataset.import", project_id=project_id,
payload={"saved": saved, "missing": len(missing)},
latency_ms=(time.perf_counter() - t0) * 1000)
return {"saved": saved, "missing": missing}
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
db: AsyncSession) -> dict:
await _require_image(project_id, image_id, db)
t0 = time.perf_counter()
await _apply_upsert(db, project_id, image_id, body)
await db.commit()
await events.emit("annotation.save", project_id=project_id, image_id=image_id,
payload={"shapeCount": len(body.shapes),

View file

@ -7,7 +7,7 @@ from fastapi import HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models import Image
from models import Image, Project
from shared import events, keys, r2
from shared.schemas import ImageConfirm
@ -40,7 +40,10 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession,
q = q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
count_q = count_q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
total = (await db.execute(count_q)).scalar() or 0
rows = (await db.execute(q.order_by(Image.created_at.desc()).limit(limit).offset(offset))).scalars().all()
# Deterministic order (id tiebreak): pages must not shift underfoot.
rows = (await db.execute(
q.order_by(Image.created_at.desc(), Image.id.desc()).limit(limit).offset(offset)
)).scalars().all()
users = (await db.execute(select(User))).scalars().all()
names = {u.id: u.name for u in users}
items = [{"id": im.id, "projectId": im.project_id, "fileName": im.file_name,
@ -57,12 +60,44 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession,
return {"items": items, "total": total}
async def locate_image(project_id: str, image_id: str, db: AsyncSession) -> dict:
"""Offset (0-based) of one image in list order, for ?imageId= deep links."""
from fastapi import HTTPException
from sqlalchemy import func, text
total = (await db.execute(
select(func.count()).where(Image.project_id == project_id))).scalar() or 0
row = (await db.execute(text(
"SELECT rn FROM (SELECT id, ROW_NUMBER() OVER "
"(ORDER BY created_at DESC, id DESC) AS rn FROM images "
"WHERE project_id = :pid) t WHERE id = :iid"),
{"pid": project_id, "iid": image_id})).first()
if row is None:
raise HTTPException(404, "image not found")
return {"offset": int(row[0]) - 1, "total": total}
def build_presign(project_id: str, file_name: str, content_type: str) -> dict:
"""Step 1/2 direct upload: frontend PUTs bytes straight to R2 with this URL."""
key = keys.image_key(project_id, "tmp", file_name) # real id assigned on confirm
import uuid as _uuid
# Unique tmp id per file: duplicate filenames in one batch must not collide.
key = keys.image_key(project_id, f"tmp-{_uuid.uuid4().hex[:8]}", file_name)
return {"putUrl": r2.presign_put(key, content_type), "r2Key": key}
async def batch_presign(project_id: str, files: list[dict], db: AsyncSession) -> list[dict]:
"""One call for a whole batch (≤200): avoids a presign roundtrip per file."""
if await db.get(Project, project_id) is None:
from fastapi import HTTPException
raise HTTPException(404, "project not found")
out = []
for f in files[:200]:
slot = build_presign(project_id, f.get("fileName", "upload.jpg"),
f.get("contentType", "image/jpeg"))
slot["fileName"] = f.get("fileName", "upload.jpg")
out.append(slot)
return out
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession) -> dict:
"""Step 2/2: metadata row after the R2 PUT succeeds."""
if not r2.head_exists(body.r2Key):

289
src/services/imports.py Normal file
View file

@ -0,0 +1,289 @@
"""Server-side dataset import: stream a big zip (GBs) without loading it to RAM.
One image in memory at a time for reads; network puts run concurrently;
DB commits in batches of 100. Re-runnable: files already in the project
(by file_name) are skipped.
"""
from __future__ import annotations
import asyncio
import json
import os
import time
import uuid
import zipfile
from fastapi import HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models import Image, ImportJob, LabelClass, Project
from shared import events, keys, r2
from shared.logger import get_logger
from shared.schemas import AnnotationShape, AnnotationUpsert, Flags
from shared.thumbnails import IMMUTABLE, resize_to_thumbnail
from src.core.db import get_session_factory
log = get_logger("danable.imports")
IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp", ".bmp")
PUT_CONCURRENCY = 24
DB_BATCH = 100
def _basename(name: str) -> str:
return name.split("/")[-1]
def _is_image(name: str) -> bool:
return name.lower().endswith(IMAGE_EXTS)
def build_bulk_from_coco(coco: dict, file_to_image: dict[str, str],
cat_to_class: dict[str, dict]) -> dict[str, AnnotationUpsert]:
"""Pure mapping: COCO annotations -> bulk-upsert payload (unit-tested)."""
cat_name = {str(c.get("id")): c.get("name", f"class_{c.get('id')}")
for c in coco.get("categories", [])}
by_image: dict[str, list] = {}
for ann in coco.get("annotations", []):
meta = next((i for i in coco.get("images", [])
if str(i.get("id")) == str(ann.get("image_id"))), None)
if meta is None:
continue
fname = _basename(meta.get("file_name", ""))
image_id = file_to_image.get(fname)
if image_id is None:
continue
label = cat_name.get(str(ann.get("category_id")), "unknown")
cls = cat_to_class.get(label.lower())
if cls is None:
continue
shape = {"labelId": cls["id"], "labelName": cls["name"], "color": cls["color"]}
seg = ann.get("segmentation")
if isinstance(seg, list) and seg and isinstance(seg[0], list) and len(seg[0]) >= 6:
ring = seg[0]
pts = [{"x": int(ring[i]), "y": int(ring[i + 1])}
for i in range(0, len(ring) - 1, 2)]
shape.update({"type": "polygon", "points": pts})
elif isinstance(ann.get("bbox"), list) and len(ann["bbox"]) == 4:
x, y, w, h = (int(v) for v in ann["bbox"])
shape.update({"type": "bbox", "bbox": {"x": x, "y": y, "width": w, "height": h}})
else:
continue # RLE/crowd without bbox: skipped
by_image.setdefault(image_id, []).append(shape)
payload: dict[str, AnnotationUpsert] = {}
for image_id, shapes in by_image.items():
payload[image_id] = AnnotationUpsert(
flags=Flags(),
shapes=[AnnotationShape(id=f"zip-{k}", index=k + 1, visible=True, **s)
for k, s in enumerate(shapes)],
)
return payload
def _job_to_dict(job: ImportJob) -> dict:
return {"id": job.id, "project_id": job.project_id, "kind": job.kind,
"status": job.status, "source": job.source,
"total_files": job.total_files, "done_files": job.done_files,
"saved_annotations": job.saved_annotations, "error": job.error}
async def list_jobs(project_id: str, db: AsyncSession) -> list[dict]:
rows = (await db.execute(select(ImportJob).where(
ImportJob.project_id == project_id).order_by(ImportJob.created_at.desc()))
).scalars().all()
return [_job_to_dict(j) for j in rows]
async def get_job(project_id: str, job_id: str, db: AsyncSession) -> dict:
job = await db.get(ImportJob, job_id)
if job is None or job.project_id != project_id:
raise HTTPException(404, "import job not found")
return _job_to_dict(job)
async def start_local_zip(project_id: str, path: str, db: AsyncSession) -> dict:
p = await db.get(Project, project_id)
if p is None:
raise HTTPException(404, "project not found")
if not os.path.isabs(path) or not path.endswith(".zip") or not os.path.exists(path):
raise HTTPException(400, "path must be an existing absolute .zip file")
job = ImportJob(project_id=project_id, kind="zip-coco", status="queued", source=path)
db.add(job)
await db.commit()
await db.refresh(job)
asyncio.create_task(_run_zip_coco(job.id, project_id, path))
return _job_to_dict(job)
async def start_r2_zip(project_id: str, r2_key: str, db: AsyncSession) -> dict:
"""Import a zip already in R2 (e.g. multipart-uploaded from the browser):
stream it to local temp (bounded RAM), run the standard pipeline, delete."""
p = await db.get(Project, project_id)
if p is None:
raise HTTPException(404, "project not found")
if not r2_key.endswith(".zip"):
raise HTTPException(400, "r2_key must point to a .zip object")
job = ImportJob(project_id=project_id, kind="zip-coco",
status="queued", source=f"r2:{r2_key}")
db.add(job)
await db.commit()
await db.refresh(job)
asyncio.create_task(_run_r2_zip(job.id, project_id, r2_key))
return _job_to_dict(job)
async def _run_r2_zip(job_id: str, project_id: str, r2_key: str) -> None:
import tempfile
tmp = tempfile.mktemp(prefix=f"danable-import-{job_id}-", suffix=".zip")
try:
await asyncio.to_thread(r2.download_to_file, r2_key, tmp)
await _run_zip_coco(job_id, project_id, tmp)
finally:
try:
os.unlink(tmp)
except Exception:
pass
async def _run_zip_coco(job_id: str, project_id: str, path: str) -> None:
t_all = time.perf_counter()
factory = get_session_factory()
async with factory() as db:
job = await db.get(ImportJob, job_id)
if job is None:
return
try:
job.status = "running"
await db.commit()
project = await db.get(Project, project_id)
if project is None:
raise RuntimeError("project gone")
zf = zipfile.ZipFile(path)
names = zf.namelist()
coco_name = next((n for n in names if _basename(n) == "annotations.json"),
next((n for n in names if n.endswith(".json")), None))
if coco_name is None:
raise RuntimeError("no COCO json found in zip")
coco = json.loads(zf.read(coco_name))
# 1. ensure label classes exist
palette = ["#3B82F6", "#10B981", "#F59E0B", "#A855F7", "#EF4444", "#06B6D4"]
existing = (await db.execute(select(LabelClass).where(
LabelClass.project_id == project_id))).scalars().all()
have = {c.name.lower() for c in existing}
for i, c in enumerate(coco.get("categories", [])):
if c.get("name", "").lower() not in have:
db.add(LabelClass(project_id=project_id, name=c["name"],
color=palette[(len(existing) + i) % len(palette)]))
have.add(c.get("name", "").lower())
await db.commit()
# 2. stream images: sequential reads, concurrent puts, batched commits
present = set((await db.execute(select(Image.file_name).where(
Image.project_id == project_id))).scalars().all())
entries = [n for n in names if _is_image(n) and _basename(n) not in present]
job.total_files = len(entries)
await db.commit()
log.info("import %s: %d new files of %d", job_id, len(entries), len(names))
dims = {_basename(i.get("file_name", "")):
(i.get("width", 0), i.get("height", 0))
for i in coco.get("images", [])}
sem = asyncio.Semaphore(PUT_CONCURRENCY)
async def process_batch(batch: list[tuple[str, bytes]]) -> None:
# Concurrent network puts (bounded); then one DB commit.
results = await asyncio.gather(*[
put_all(
keys.image_key(project.slug, f"zip-{uuid.uuid4().hex[:8]}", fname),
keys.thumbnail_key(project.slug, fname.rsplit(".", 1)[0]),
keys.thumbnail_small_key(project.slug, fname.rsplit(".", 1)[0]),
data,
)
for fname, data in batch
])
rows = []
for (fname, data), keys3 in zip(batch, results):
r2_key, thumb_key, small_key = keys3
w, h = dims.get(fname, (0, 0))
rows.append(Image(
project_id=project_id, file_name=fname, r2_key=r2_key,
thumbnail_r2_key=thumb_key, thumbnail_small_r2_key=small_key,
width=w, height=h, status="unassigned", flags={}))
db.add_all(rows)
await db.commit()
job.done_files = (job.done_files or 0) + len(rows)
await db.commit()
async def put_all(r2_key: str, thumb_key: str, small_key: str,
data: bytes) -> tuple[str, str, str]:
async with sem:
await asyncio.to_thread(r2.put_bytes, r2_key, data, "image/jpeg")
try:
big = await asyncio.to_thread(resize_to_thumbnail, data, 512, 70)
tiny = await asyncio.to_thread(resize_to_thumbnail, data, 128, 65)
await asyncio.to_thread(
r2.put_bytes, thumb_key, big, "image/jpeg", IMMUTABLE)
await asyncio.to_thread(
r2.put_bytes, small_key, tiny, "image/jpeg", IMMUTABLE)
except Exception as e:
log.warning("thumb inline failed: %s", e)
return (r2_key, thumb_key, small_key)
batch: list[tuple[str, str]] = []
async def flush_names() -> None:
if not batch:
return
# Sequential disk reads (zipfile is not thread-safe), then
# concurrent network puts for the whole batch at once.
loaded = [(fname, zf.read(name)) for name, fname in batch]
await process_batch(loaded)
batch.clear()
for name in entries:
batch.append((name, _basename(name)))
if len(batch) >= 48:
await flush_names()
await flush_names()
# 3. annotations via the bulk path in 500-image chunks
rows = (await db.execute(select(Image.id, Image.file_name).where(
Image.project_id == project_id))).all()
file_to_image = {fname: iid for iid, fname in rows}
classes = (await db.execute(select(LabelClass).where(
LabelClass.project_id == project_id))).scalars().all()
cat_to_class = {c.name.lower(): {"id": c.id, "name": c.name, "color": c.color}
for c in classes}
from src.services import annotations as ann_svc
payload = build_bulk_from_coco(coco, file_to_image, cat_to_class)
saved = 0
chunk: dict[str, AnnotationUpsert] = {}
for iid, body in payload.items():
chunk[iid] = body
if len(chunk) >= 500:
out = await ann_svc.bulk_upsert(project_id, chunk, db)
saved += out["saved"]
chunk = {}
if chunk:
out = await ann_svc.bulk_upsert(project_id, chunk, db)
saved += out["saved"]
job.saved_annotations = saved
job.status = "done"
await db.commit()
await events.emit("dataset.import", project_id=project_id,
payload={"via": "zip", "files": job.done_files, "saved": saved},
latency_ms=(time.perf_counter() - t_all) * 1000)
log.info("import %s done: %d files, %d annotations", job_id, job.done_files, saved)
except Exception as e:
log.warning("import %s failed: %s", job_id, e)
try:
job = await db.get(ImportJob, job_id)
if job is not None:
job.status = "error"
job.error = str(e)[:2000]
await db.commit()
except Exception:
pass

View file

@ -44,16 +44,23 @@ async def create_project(body: ProjectCreate, db: AsyncSession) -> dict:
async def _stats(project_id: str, db: AsyncSession) -> dict:
counts: dict[str, int] = {}
for st in ("unassigned", "in_progress", "pending_review", "completed"):
counts[st] = (await db.execute(
select(func.count()).where(Image.project_id == project_id, Image.status == st))).scalar() or 0
"""One GROUP BY instead of five counts + a sum: the list endpoint calls
this per project, so roundtrips dominate at 10+ projects."""
rows = (await db.execute(
select(Image.status, func.count(),
func.coalesce(func.sum(Image.object_count), 0))
.where(Image.project_id == project_id)
.group_by(Image.status))).all()
counts = {st: 0 for st in ("unassigned", "in_progress", "pending_review", "completed")}
objects = 0
for st, n, o in rows:
if st in counts:
counts[st] = n or 0
objects += int(o or 0)
total = sum(counts.values())
objects = (await db.execute(
select(func.coalesce(func.sum(Image.object_count), 0)).where(Image.project_id == project_id))).scalar() or 0
return {"totalImages": total, "unassigned": counts["unassigned"], "inProgress": counts["in_progress"],
"pendingReview": counts["pending_review"], "completed": counts["completed"],
"totalObjects": int(objects)}
"totalObjects": objects}
async def _members(project_id: str, db: AsyncSession) -> list[dict]:
@ -94,13 +101,28 @@ async def _thumbnail_url(project_id: str, db: AsyncSession) -> str:
return ""
async def _class_counts(project_id: str, db: AsyncSession) -> dict[str, int]:
"""Per-label object totals straight from stored shapes (always exact,
including bulk imports and deletions — no counter drift possible)."""
from sqlalchemy import text
rows = (await db.execute(text(
"SELECT elem->>'labelId' AS lid, COUNT(*) FROM annotations, "
"LATERAL json_array_elements(CASE WHEN shapes IS NULL THEN '[]' "
"WHEN json_typeof(shapes) = 'array' THEN shapes ELSE '[]' END) AS elem "
"WHERE project_id = :pid GROUP BY lid"),
{"pid": project_id})).all()
return {lid: n for lid, n in rows if lid}
async def _project_card(p: Project, db: AsyncSession) -> dict:
classes = (await db.execute(select(LabelClass).where(LabelClass.project_id == p.id))).scalars().all()
counts = await _class_counts(p.id, db)
return {"id": p.id, "slug": p.slug, "name": p.name, "taskType": p.task_type,
"status": p.status, "description": p.description,
"thumbnailUrl": await _thumbnail_url(p.id, db),
"updatedAt": p.updated_at.isoformat() if p.updated_at else "",
"classes": [{"id": c.id, "name": c.name, "color": c.color} for c in classes],
"classes": [{"id": c.id, "name": c.name, "color": c.color,
"count": counts.get(c.id, 0)} for c in classes],
"stats": await _stats(p.id, db),
"members": await _members(p.id, db)}

View file

@ -41,6 +41,17 @@ async def invite_user(email: str, role: str, project_ids: list[str], db: AsyncSe
return to_public(u)
async def delete_user(user_id: str, actor_id: str | None, db: AsyncSession) -> dict:
u = await db.get(User, user_id)
if u is None:
raise HTTPException(404, "user not found")
if actor_id is not None and u.id == actor_id:
raise HTTPException(400, "cannot delete your own account")
await db.delete(u)
await db.commit()
return {"deleted": user_id}
async def patch_user(user_id: str, patch: dict, db: AsyncSession) -> dict:
u = await db.get(User, user_id)
if u is None:

84
tests/test_bulk.py Normal file
View file

@ -0,0 +1,84 @@
"""Batch upload prep + bulk annotation import (COCO/YOLO path)."""
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from shared.schemas import AnnotationUpsert
from src.services import annotations as ann_svc
from src.services import images as img_svc
@pytest_asyncio.fixture
async def db():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
from src.core.db import Base
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
yield async_sessionmaker(engine, expire_on_commit=False)
await engine.dispose()
async def _seed(db):
async with db() as s:
await s.execute(text(
"INSERT INTO projects (id, slug, name, task_type, status) "
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
await s.execute(text(
"INSERT INTO images (id, project_id, file_name, r2_key, status, flags) "
"VALUES ('img1', 'p1', 'a.jpg', 'k1', 'in_progress', '{}'),"
"('img2', 'p1', 'b.jpg', 'k2', 'in_progress', '{}')"))
await s.commit()
def _body(n=1):
from shared.schemas import AnnotationShape
return AnnotationUpsert(shapes=[
AnnotationShape(id=f"s{i}", index=i + 1, type="bbox", labelId="car",
bbox={"x": 1, "y": 2, "width": 10, "height": 10})
for i in range(n)])
@pytest.mark.asyncio
async def test_bulk_upsert_saves_and_reports_missing(db):
await _seed(db)
async with db() as s:
out = await ann_svc.bulk_upsert("p1", {"img1": _body(2), "ghost": _body(1)}, s)
assert out == {"saved": 1, "missing": ["ghost"]}
n = (await s.execute(text("SELECT object_count FROM images WHERE id='img1'"))).scalar()
assert n == 2
got = await ann_svc.get_annotation("p1", "img1", s)
assert len(got["shapes"]) == 2
st = (await s.execute(text("SELECT status FROM images WHERE id='img1'"))).scalar()
assert st == "pending_review"
@pytest.mark.asyncio
async def test_bulk_upsert_unknown_project_404(db):
from fastapi import HTTPException
await _seed(db)
async with db() as s:
with pytest.raises(HTTPException) as e:
await ann_svc.bulk_upsert("nope", {"img1": _body()}, s)
assert e.value.status_code == 404
@pytest.mark.asyncio
async def test_batch_presign_unique_keys_and_cap(db):
await _seed(db)
async with db() as s:
files = [{"fileName": "same.jpg", "contentType": "image/jpeg"} for _ in range(3)]
out = await img_svc.batch_presign("p1", files, s)
keys = [o["r2Key"] for o in out]
assert len(set(keys)) == 3 and all("same.jpg" in k for k in keys)
assert all(o["putUrl"] for o in out)
@pytest.mark.asyncio
async def test_batch_presign_unknown_project_404(db):
from fastapi import HTTPException
await _seed(db)
async with db() as s:
with pytest.raises(HTTPException) as e:
await img_svc.batch_presign("nope", [{"fileName": "a.jpg"}], s)
assert e.value.status_code == 404

59
tests/test_imports.py Normal file
View file

@ -0,0 +1,59 @@
"""Server zip import: pure COCO mapping + job validation (no R2 touched)."""
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from src.services.imports import build_bulk_from_coco
from src.services import imports as svc
@pytest_asyncio.fixture
async def db():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
from src.core.db import Base
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
yield async_sessionmaker(engine, expire_on_commit=False)
await engine.dispose()
COCO = {
"images": [{"id": 1, "file_name": "a.jpg"}, {"id": 2, "file_name": "b.jpg"}],
"annotations": [
{"image_id": 1, "category_id": 1, "bbox": [10, 20, 30, 40]},
{"image_id": 1, "category_id": 2,
"segmentation": [[0, 0, 10, 0, 10, 10, 0, 10]]},
{"image_id": 2, "category_id": 9, "iscrowd": 1,
"segmentation": {"counts": "xyz"}},
{"image_id": 999, "category_id": 1, "bbox": [0, 0, 5, 5]},
],
"categories": [{"id": 1, "name": "car"}, {"id": 2, "name": "bus"}],
}
CATS = {"car": {"id": "c1", "name": "car", "color": "#fff"},
"bus": {"id": "c2", "name": "bus", "color": "#000"}}
def test_bulk_mapping_bbox_polygon_and_skips():
payload = build_bulk_from_coco(
COCO, {"a.jpg": "img1", "b.jpg": "img2"}, CATS)
assert set(payload) == {"img1"} # b.jpg: RLE-only + unknown image 999 skipped
kinds = sorted(s.type for s in payload["img1"].shapes)
assert kinds == ["bbox", "polygon"]
poly = next(s for s in payload["img1"].shapes if s.type == "polygon")
assert [(p.x, p.y) for p in poly.points or []] == [(0, 0), (10, 0), (10, 10), (0, 10)]
assert payload["img1"].shapes[0].labelId == "c1"
@pytest.mark.asyncio
async def test_start_local_zip_validates(db):
from fastapi import HTTPException
async with db() as s:
await s.execute(text(
"INSERT INTO projects (id, slug, name, task_type, status) "
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
await s.commit()
with pytest.raises(HTTPException):
await svc.start_local_zip("p1", "relative.zip", s)
with pytest.raises(HTTPException):
await svc.start_local_zip("nope", "/tmp/x.zip", s)

View file

@ -54,6 +54,26 @@ async def test_status_and_search_filter_server_side(db):
assert one["total"] == 1 and one["items"][0]["id"] == "img1"
@pytest.mark.asyncio
async def test_locate_image_offset_matches_pages(db):
await _seed(db, 5)
async with db() as s:
loc = await svc.locate_image("p1", "img2", s)
assert loc["total"] == 5
page = await svc.list_images("p1", "all", s, limit=2, offset=loc["offset"])
assert page["items"][0]["id"] == "img2"
@pytest.mark.asyncio
async def test_locate_missing_image_is_404(db):
from fastapi import HTTPException
await _seed(db, 1)
async with db() as s:
with pytest.raises(HTTPException) as e:
await svc.locate_image("p1", "nope", s)
assert e.value.status_code == 404
@pytest.mark.asyncio
async def test_limit_is_clamped(db):
await _seed(db, 2)