diff --git a/config.example.yaml b/config.example.yaml index 36346d8..d31bd48 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -15,6 +15,8 @@ database: storage: bucket_default: danagble + # Reference quota for the sidebar meter (R2 free tier = 10 GiB). + quota_bytes: 10737418240 # R2 key layout. {project} = project slug, {image} = image id. layout: image: "datasets/{project}/{image}-{filename}" diff --git a/config.yaml b/config.yaml index 36346d8..d31bd48 100644 --- a/config.yaml +++ b/config.yaml @@ -15,6 +15,8 @@ database: storage: bucket_default: danagble + # Reference quota for the sidebar meter (R2 free tier = 10 GiB). + quota_bytes: 10737418240 # R2 key layout. {project} = project slug, {image} = image id. layout: image: "datasets/{project}/{image}-{filename}" diff --git a/docs/API.md b/docs/API.md index 57006f3..8579f64 100644 --- a/docs/API.md +++ b/docs/API.md @@ -24,6 +24,7 @@ Interactive docs: `http://localhost:8000/docs`. |---|---|---| | GET | `/health` | Liveness incl. DB check, `{ ok: true }` | | GET | `/metrics` | Phase-1 traffic counters per route | +| GET | `/storage/usage` | Real bucket `{ bytes, objects, quota_bytes }` (10-min cache) | | GET | `/` (root, no prefix) | Name, docs and health links | | POST | `/admin/init-db` | Create tables, seed dev admin. Dev-only | | POST | `/admin/backfill-thumbnails?limit=100` | Mint missing thumbs. Dev-only | @@ -97,6 +98,44 @@ GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0 `PATCH /projects/{id}/images/{imageId}?status=completed` — status transition. +`POST /projects/{id}/images/batch-presign` — one call for ≤200 files: +`[{ fileName, contentType }]` → `[{ fileName, putUrl, r2Key }]` +(keys carry unique tmp ids, so duplicate filenames never collide). + +`GET /projects/{id}/images/by-id/{imageId}` — `{ offset, total }` in list +order (deterministic `created_at DESC, id DESC`); backs `?imageId=` deep +links into thousand-image datasets. + +## Resumable multipart uploads (files ~100MB and up) + +Small files keep the single-PUT presign path. Big single files +(GB zips, TIFF, video) go multipart, straight to R2: + +1. `POST /projects/{id}/uploads/multipart` `{ fileName, contentType }` → + `{ key, uploadId, partSize: 8388608 }`. +2. `POST .../uploads/multipart/parts` `{ key, uploadId, numbers: [1..N] }` + → `{ urls: { "1": "...", ... } }` (1h TTL each). +3. PUT every part (parallel, per-part retry; store ETags client-side — + the upload resumes after network drops instead of restarting). +4. `POST .../uploads/multipart/complete` `{ key, uploadId, + parts: [{ ETag, PartNumber }] }` → `{ key, etag }`. +5. `POST .../uploads/multipart/abort` to cancel and free the upload. + +ETag response headers are exposed via the bucket CORS policy, which the +frontend needs to read them. + +## Server-side dataset imports (GB-scale zips) + +`POST /projects/{id}/imports/local-zip` `{ path }` (absolute server path) +or `POST /projects/{id}/imports/r2-zip` `{ r2_key }` (zip already uploaded, +e.g. via multipart) → `202 { id, status: queued, ... }`. The worker streams +the archive (one image in RAM at a time), uploads originals + 512/128px +thumbs concurrently, commits DB rows in batches, then bulk-upserts COCO +annotations. Re-running skips files already present. + +`GET /projects/{id}/imports` and `GET /projects/{id}/imports/{jobId}` poll +`{ status, total_files, done_files, saved_annotations, error }`. + ## Annotations (auth) `GET /projects/{pid}/images/{iid}/annotation` — one row per image: @@ -109,6 +148,10 @@ GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0 "visible": true, "bbox": { "x": 10, "y": 20, "width": 100, "height": 80 } }] } ``` +`PUT /projects/{id}/annotations/batch` with `{ [imageId]: { classificationLabelId?, flags?, shapes[] } }` +upserts up to 500 images in one commit (COCO/YOLO import path) → +`{ saved, missing: [...] }`. + `PUT` same path with `{ classificationLabelId?, flags?, shapes[] }` upserts (one row per image, 404 when the image does not exist) and syncs `object_count` / flags onto the image row. Also emits an `annotation.save` diff --git a/models/__init__.py b/models/__init__.py index 0e6bc4e..b755704 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -5,10 +5,11 @@ from models.tables import ( Annotation, ExportJob, Image, + ImportJob, LabelClass, Project, User, ) __all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image", - "LabelClass", "Project", "User"] + "ImportJob", "LabelClass", "Project", "User"] diff --git a/models/tables.py b/models/tables.py index 17315f7..97d2598 100644 --- a/models/tables.py +++ b/models/tables.py @@ -104,6 +104,22 @@ class ActionEvent(Base): created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) +class ImportJob(Base): + """Server-side dataset import (e.g. 3GB zip): polled by the UI/admin.""" + __tablename__ = "imports" + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid) + project_id: Mapped[str] = mapped_column(String(32), index=True) + kind: Mapped[str] = mapped_column(String(32), default="zip-coco") + status: Mapped[str] = mapped_column(String(32), default="queued", index=True) + source: Mapped[str] = mapped_column(String(1024), default="") + total_files: Mapped[int] = mapped_column(Integer, default=0) + done_files: Mapped[int] = mapped_column(Integer, default=0) + saved_annotations: Mapped[int] = mapped_column(Integer, default=0) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + class ExportJob(Base): __tablename__ = "exports" id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid) diff --git a/shared/r2.py b/shared/r2.py index aac12a1..85fab0d 100644 --- a/shared/r2.py +++ b/shared/r2.py @@ -64,6 +64,67 @@ def get_bytes(key: str) -> bytes: return obj["Body"].read() +def bucket_usage(prefix: str = "") -> dict: + """Sum bytes + object count via paginated LIST (no download).""" + s = get_settings() + total_bytes, total_objects = 0, 0 + paginator = _client().get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=s.r2_bucket, Prefix=prefix): + for obj in page.get("Contents", []): + total_bytes += obj.get("Size", 0) + total_objects += 1 + return {"bytes": total_bytes, "objects": total_objects} + + +def create_multipart(key: str, content_type: str = "application/octet-stream") -> str: + """Start a multipart upload; returns upload_id. Parts upload in parallel, + each retriable; the upload survives network drops via the stored id.""" + s = get_settings() + res = _client().create_multipart_upload( + Bucket=s.r2_bucket, Key=key, ContentType=content_type) + return res["UploadId"] + + +def presign_part(key: str, upload_id: str, part_number: int, + expires_in: int = 3600) -> str: + s = get_settings() + return _client().generate_presigned_url( + "upload_part", + Params={"Bucket": s.r2_bucket, "Key": key, + "UploadId": upload_id, "PartNumber": part_number}, + ExpiresIn=expires_in) + + +def complete_multipart(key: str, upload_id: str, parts: list[dict]) -> str: + """parts: [{ETag, PartNumber}]. Returns final ETag.""" + s = get_settings() + res = _client().complete_multipart_upload( + Bucket=s.r2_bucket, Key=key, UploadId=upload_id, + MultipartUpload={"Parts": [{"ETag": p["ETag"], "PartNumber": p["PartNumber"]} + for p in sorted(parts, key=lambda x: x["PartNumber"])]}) + return res.get("ETag", "") + + +def abort_multipart(key: str, upload_id: str) -> None: + s = get_settings() + try: + _client().abort_multipart_upload(Bucket=s.r2_bucket, Key=key, UploadId=upload_id) + except Exception: + pass + + +def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int: + """Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes.""" + s = get_settings() + body = _client().get_object(Bucket=s.r2_bucket, Key=key)["Body"] + total = 0 + with open(dest_path, "wb") as f: + for chunk in body.iter_chunks(chunk_size=chunk_bytes): + f.write(chunk) + total += len(chunk) + return total + + def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg", cache_control: str | None = None) -> None: s = get_settings() diff --git a/shared/schemas.py b/shared/schemas.py index 8318030..d387610 100644 --- a/shared/schemas.py +++ b/shared/schemas.py @@ -73,3 +73,14 @@ class ImageConfirm(BaseModel): class PresignRequest(BaseModel): fileName: str contentType: str = "image/jpeg" + + +class BatchPresignItem(BaseModel): + fileName: str + contentType: str = "image/jpeg" + + +class BulkAnnotationItem(BaseModel): + classificationLabelId: str | None = None + flags: Flags = Field(default_factory=Flags) + shapes: list[AnnotationShape] = Field(default_factory=list) diff --git a/src/api/v1/auth.py b/src/api/v1/auth.py index 58baae2..38bce2d 100644 --- a/src/api/v1/auth.py +++ b/src/api/v1/auth.py @@ -71,3 +71,11 @@ async def invite(body: InviteIn, db: AsyncSession = Depends(get_session)): async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)): from src.services import users as svc return await svc.patch_user(user_id, body.model_dump(), db) + + +@users_router.delete("/{user_id}") +async def delete_user(user_id: str, db: AsyncSession = Depends(get_session), + user = Depends(get_current_user)): + from src.services import users as svc + actor = user.id if user is not None else None + return await svc.delete_user(user_id, actor, db) diff --git a/src/api/v1/images.py b/src/api/v1/images.py index f6cfeb3..7d6a295 100644 --- a/src/api/v1/images.py +++ b/src/api/v1/images.py @@ -3,13 +3,21 @@ from __future__ import annotations from fastapi import APIRouter, Depends, Query from sqlalchemy.ext.asyncio import AsyncSession -from shared.schemas import ImageConfirm, PresignRequest +from shared.schemas import BatchPresignItem, ImageConfirm, PresignRequest from src.core.db import get_session from src.services import images as svc router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"]) +@router.get("/by-id/{image_id}") +async def locate_image(project_id: str, image_id: str, + db: AsyncSession = Depends(get_session)): + """Deep-link support (?imageId=): offset of one image in list order.""" + from src.services import images as svc + return await svc.locate_image(project_id, image_id, db) + + @router.get("") async def list_images(project_id: str, status: str | None = Query(default=None), search: str | None = Query(default=None), @@ -26,6 +34,13 @@ async def presign_upload(project_id: str, body: PresignRequest): return svc.build_presign(project_id, body.fileName, body.contentType) +@router.post("/batch-presign") +async def batch_presign_upload(project_id: str, body: list[BatchPresignItem], + db: AsyncSession = Depends(get_session)): + """One call for a whole upload batch (≤200 files).""" + return await svc.batch_presign(project_id, [b.model_dump() for b in body], db) + + @router.post("", status_code=201) async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)): return await svc.confirm_image(project_id, body, db) diff --git a/src/api/v1/imports.py b/src/api/v1/imports.py new file mode 100644 index 0000000..fc55ff7 --- /dev/null +++ b/src/api/v1/imports.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends +from pydantic import BaseModel +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db import get_session +from src.services import imports as svc + +router = APIRouter(prefix="/projects/{project_id}/imports", tags=["imports"]) + + +class LocalZipIn(BaseModel): + path: str + + +class R2ZipIn(BaseModel): + r2_key: str + + +@router.post("/local-zip", status_code=202) +async def start_local_zip(project_id: str, body: LocalZipIn, + db: AsyncSession = Depends(get_session)): + """Queue a server-side zip import (GB-scale, streamed). Poll the job id.""" + return await svc.start_local_zip(project_id, body.path, db) + + +@router.post("/r2-zip", status_code=202) +async def start_r2_zip(project_id: str, body: R2ZipIn, + db: AsyncSession = Depends(get_session)): + """Queue import of a zip already uploaded to R2 (e.g. via multipart).""" + return await svc.start_r2_zip(project_id, body.r2_key, db) + + +@router.get("") +async def list_imports(project_id: str, db: AsyncSession = Depends(get_session)): + return await svc.list_jobs(project_id, db) + + +@router.get("/{job_id}") +async def get_import(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)): + return await svc.get_job(project_id, job_id, db) diff --git a/src/api/v1/projects.py b/src/api/v1/projects.py index 324f357..7e35ba8 100644 --- a/src/api/v1/projects.py +++ b/src/api/v1/projects.py @@ -3,8 +3,9 @@ from __future__ import annotations from fastapi import APIRouter, Depends from sqlalchemy.ext.asyncio import AsyncSession -from shared.schemas import LabelClassIn, ProjectCreate +from shared.schemas import AnnotationUpsert, BulkAnnotationItem, LabelClassIn, ProjectCreate from src.core.db import get_session +from src.services import annotations as annotations_svc from src.services import projects as svc router = APIRouter(prefix="/projects", tags=["projects"]) @@ -35,6 +36,14 @@ async def delete_project(project_id: str, db: AsyncSession = Depends(get_session return await svc.delete_project(project_id, db) +@router.put("/{project_id}/annotations/batch") +async def bulk_upsert_annotations(project_id: str, body: dict[str, BulkAnnotationItem], + db: AsyncSession = Depends(get_session)): + """COCO/YOLO import: upsert up to 500 images in one commit.""" + items = {image_id: AnnotationUpsert(**item.model_dump()) for image_id, item in body.items()} + return await annotations_svc.bulk_upsert(project_id, items, db) + + @router.post("/{project_id}/classes", status_code=201) async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)): return await svc.add_label_class(project_id, body, db) diff --git a/src/api/v1/system.py b/src/api/v1/system.py index bb6fa87..24d248a 100644 --- a/src/api/v1/system.py +++ b/src/api/v1/system.py @@ -1,6 +1,6 @@ from __future__ import annotations -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from sqlalchemy import select, text from sqlalchemy.ext.asyncio import AsyncSession @@ -22,6 +22,28 @@ async def metrics(): return get_snapshot() +_usage_cache: dict = {"at": 0.0, "data": None} + + +@router.get("/storage/usage") +async def storage_usage(): + """Real bucket bytes/objects for the sidebar meter (10-min cache).""" + import time + from shared import r2 + from src.core.config import get_settings + now = time.time() + if _usage_cache["data"] is not None and now - _usage_cache["at"] < 600: + return _usage_cache["data"] + try: + usage = r2.bucket_usage() + except Exception as e: + raise HTTPException(503, f"storage unreachable: {e}") + data = {"bytes": usage["bytes"], "objects": usage["objects"], + "quota_bytes": get_settings().storage_quota_bytes} + _usage_cache.update({"at": now, "data": data}) + return data + + @router.post("/admin/backfill-thumbnails") async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)): """Mint thumbs for images uploaded before thumbnailing existed. Dev-only.""" diff --git a/src/api/v1/uploads.py b/src/api/v1/uploads.py new file mode 100644 index 0000000..d85c1c7 --- /dev/null +++ b/src/api/v1/uploads.py @@ -0,0 +1,102 @@ +"""Resumable multipart uploads: big single files (GB zip/TIFF/video). + +Flow: POST /multipart -> {key, uploadId} -> PUT each part to its presigned +URL (parallel, retried per part, resumable via stored ETags) -> +POST /multipart/complete. Use for anything over ~100MB; small files keep +the single-PUT presign path. +""" +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from models import Project +from shared import r2 +from src.core.db import get_session + +router = APIRouter(prefix="/projects/{project_id}/uploads", tags=["uploads"]) + +PART_SIZE = 8 * 1024 * 1024 + + +class MultipartStart(BaseModel): + fileName: str + contentType: str = "application/octet-stream" + + +class MultipartParts(BaseModel): + key: str + uploadId: str + numbers: list[int] = Field(max_length=1000) + + +class PartETag(BaseModel): + ETag: str + PartNumber: int + + +class MultipartComplete(BaseModel): + key: str + uploadId: str + parts: list[PartETag] + + +class MultipartAbort(BaseModel): + key: str + uploadId: str + + +async def _project_slug(project_id: str, db: AsyncSession) -> str: + p = await db.get(Project, project_id) + if p is None: + raise HTTPException(404, "project not found") + return p.slug + + +@router.post("/multipart") +async def start_multipart(project_id: str, body: MultipartStart, + db: AsyncSession = Depends(get_session)): + import uuid as _uuid + slug = await _project_slug(project_id, db) + key = f"uploads/{slug}/{_uuid.uuid4().hex[:12]}-{body.fileName.replace('/', '_')}" + try: + upload_id = r2.create_multipart(key, body.contentType) + except Exception as e: + raise HTTPException(503, f"storage unreachable: {e}") + return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE} + + +@router.post("/multipart/parts") +async def part_urls(project_id: str, body: MultipartParts, + db: AsyncSession = Depends(get_session)): + await _project_slug(project_id, db) + if any(n < 1 or n > 10000 for n in body.numbers): + raise HTTPException(400, "invalid part numbers") + try: + urls = {n: r2.presign_part(body.key, body.uploadId, n) for n in body.numbers} + except Exception as e: + raise HTTPException(503, f"storage unreachable: {e}") + return {"urls": urls} + + +@router.post("/multipart/complete") +async def complete_multipart(project_id: str, body: MultipartComplete, + db: AsyncSession = Depends(get_session)): + await _project_slug(project_id, db) + try: + etag = r2.complete_multipart(body.key, body.uploadId, + [p.model_dump() for p in body.parts]) + except Exception as e: + raise HTTPException(400, f"complete failed (missing/extra parts?): {e}") + if not r2.head_exists(body.key): + raise HTTPException(400, "object not found after complete") + return {"key": body.key, "etag": etag} + + +@router.post("/multipart/abort") +async def abort_multipart(project_id: str, body: MultipartAbort, + db: AsyncSession = Depends(get_session)): + await _project_slug(project_id, db) + r2.abort_multipart(body.key, body.uploadId) + return {"aborted": True} diff --git a/src/core/config.py b/src/core/config.py index d6728e8..7c45802 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -46,6 +46,7 @@ class Settings: layout = storage.get("layout", {}) presigned = storage.get("presigned_ttl", {}) self.r2_layout: dict = layout + self.storage_quota_bytes: int = int(storage.get("quota_bytes", 10 * 1024**3)) thumb = storage.get("thumbnail", {}) self.thumbnail_enabled: bool = bool(thumb.get("enabled", True)) self.thumbnail_max_side: int = int(thumb.get("max_side", 512)) diff --git a/src/main.py b/src/main.py index 6668f54..13050a4 100644 --- a/src/main.py +++ b/src/main.py @@ -11,7 +11,7 @@ from shared import events from shared.logger import get_logger from shared.tracing import TraceMiddleware from shared.traffic import TrafficMiddleware -from src.api.v1 import activities, annotations, auth, events as events_api, images, projects, system +from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads from src.api.v1.auth import users_router from src.core.config import get_settings from src.core.security import get_current_user @@ -61,6 +61,8 @@ app.include_router(activities.router, prefix=prefix, dependencies=authed) app.include_router(projects.router, prefix=prefix, dependencies=authed) app.include_router(images.router, prefix=prefix, dependencies=authed) app.include_router(annotations.router, prefix=prefix, dependencies=authed) +app.include_router(imports.router, prefix=prefix, dependencies=authed) +app.include_router(uploads.router, prefix=prefix, dependencies=authed) @app.get("/") diff --git a/src/services/annotations.py b/src/services/annotations.py index ad49cdf..a40b41d 100644 --- a/src/services/annotations.py +++ b/src/services/annotations.py @@ -6,7 +6,7 @@ import time from fastapi import HTTPException from sqlalchemy.ext.asyncio import AsyncSession -from models import Annotation, Image +from models import Annotation, Image, Project from shared import events from shared.schemas import AnnotationUpsert @@ -29,10 +29,8 @@ async def get_annotation(project_id: str, image_id: str, db: AsyncSession) -> di "flags": ann.flags, "shapes": ann.shapes} -async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert, - db: AsyncSession) -> dict: - im = await _require_image(project_id, image_id, db) - t0 = time.perf_counter() +async def _apply_upsert(db: AsyncSession, project_id: str, image_id: str, + body: AnnotationUpsert) -> int: ann = await db.get(Annotation, image_id) payload = {"project_id": project_id, "classification_label_id": body.classificationLabelId, @@ -49,6 +47,43 @@ async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpse im.object_count = len(body.shapes) im.classification_label_id = body.classificationLabelId im.flags = body.flags.model_dump() + return len(body.shapes) + + +async def bulk_upsert(project_id: str, items: dict[str, AnnotationUpsert], + db: AsyncSession) -> dict: + """COCO/YOLO import path: one commit for the whole batch (≤500 images). + + Imported labels are machine data awaiting human review, so saved rows go + to pending_review — that is what the review queue and stats count on. + """ + from fastapi import HTTPException + if await db.get(Project, project_id) is None: + raise HTTPException(404, "project not found") + from shared import events + import time + saved, missing = 0, [] + t0 = time.perf_counter() + for image_id, body in list(items.items())[:500]: + im = await db.get(Image, image_id) + if im is None or im.project_id != project_id: + missing.append(image_id) + continue + await _apply_upsert(db, project_id, image_id, body) + im.status = "pending_review" + saved += 1 + await db.commit() + await events.emit("dataset.import", project_id=project_id, + payload={"saved": saved, "missing": len(missing)}, + latency_ms=(time.perf_counter() - t0) * 1000) + return {"saved": saved, "missing": missing} + + +async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert, + db: AsyncSession) -> dict: + await _require_image(project_id, image_id, db) + t0 = time.perf_counter() + await _apply_upsert(db, project_id, image_id, body) await db.commit() await events.emit("annotation.save", project_id=project_id, image_id=image_id, payload={"shapeCount": len(body.shapes), diff --git a/src/services/images.py b/src/services/images.py index 60b4ae3..24d0af0 100644 --- a/src/services/images.py +++ b/src/services/images.py @@ -7,7 +7,7 @@ from fastapi import HTTPException from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from models import Image +from models import Image, Project from shared import events, keys, r2 from shared.schemas import ImageConfirm @@ -40,7 +40,10 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession, q = q.join(User, User.id == Image.assigned_to).where(User.name == assignee) count_q = count_q.join(User, User.id == Image.assigned_to).where(User.name == assignee) total = (await db.execute(count_q)).scalar() or 0 - rows = (await db.execute(q.order_by(Image.created_at.desc()).limit(limit).offset(offset))).scalars().all() + # Deterministic order (id tiebreak): pages must not shift underfoot. + rows = (await db.execute( + q.order_by(Image.created_at.desc(), Image.id.desc()).limit(limit).offset(offset) + )).scalars().all() users = (await db.execute(select(User))).scalars().all() names = {u.id: u.name for u in users} items = [{"id": im.id, "projectId": im.project_id, "fileName": im.file_name, @@ -57,12 +60,44 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession, return {"items": items, "total": total} +async def locate_image(project_id: str, image_id: str, db: AsyncSession) -> dict: + """Offset (0-based) of one image in list order, for ?imageId= deep links.""" + from fastapi import HTTPException + from sqlalchemy import func, text + total = (await db.execute( + select(func.count()).where(Image.project_id == project_id))).scalar() or 0 + row = (await db.execute(text( + "SELECT rn FROM (SELECT id, ROW_NUMBER() OVER " + "(ORDER BY created_at DESC, id DESC) AS rn FROM images " + "WHERE project_id = :pid) t WHERE id = :iid"), + {"pid": project_id, "iid": image_id})).first() + if row is None: + raise HTTPException(404, "image not found") + return {"offset": int(row[0]) - 1, "total": total} + + def build_presign(project_id: str, file_name: str, content_type: str) -> dict: """Step 1/2 direct upload: frontend PUTs bytes straight to R2 with this URL.""" - key = keys.image_key(project_id, "tmp", file_name) # real id assigned on confirm + import uuid as _uuid + # Unique tmp id per file: duplicate filenames in one batch must not collide. + key = keys.image_key(project_id, f"tmp-{_uuid.uuid4().hex[:8]}", file_name) return {"putUrl": r2.presign_put(key, content_type), "r2Key": key} +async def batch_presign(project_id: str, files: list[dict], db: AsyncSession) -> list[dict]: + """One call for a whole batch (≤200): avoids a presign roundtrip per file.""" + if await db.get(Project, project_id) is None: + from fastapi import HTTPException + raise HTTPException(404, "project not found") + out = [] + for f in files[:200]: + slot = build_presign(project_id, f.get("fileName", "upload.jpg"), + f.get("contentType", "image/jpeg")) + slot["fileName"] = f.get("fileName", "upload.jpg") + out.append(slot) + return out + + async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession) -> dict: """Step 2/2: metadata row after the R2 PUT succeeds.""" if not r2.head_exists(body.r2Key): diff --git a/src/services/imports.py b/src/services/imports.py new file mode 100644 index 0000000..00d2d82 --- /dev/null +++ b/src/services/imports.py @@ -0,0 +1,289 @@ +"""Server-side dataset import: stream a big zip (GBs) without loading it to RAM. + +One image in memory at a time for reads; network puts run concurrently; +DB commits in batches of 100. Re-runnable: files already in the project +(by file_name) are skipped. +""" +from __future__ import annotations + +import asyncio +import json +import os +import time +import uuid +import zipfile + +from fastapi import HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from models import Image, ImportJob, LabelClass, Project +from shared import events, keys, r2 +from shared.logger import get_logger +from shared.schemas import AnnotationShape, AnnotationUpsert, Flags +from shared.thumbnails import IMMUTABLE, resize_to_thumbnail +from src.core.db import get_session_factory + +log = get_logger("danable.imports") + +IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp", ".bmp") +PUT_CONCURRENCY = 24 +DB_BATCH = 100 + + +def _basename(name: str) -> str: + return name.split("/")[-1] + + +def _is_image(name: str) -> bool: + return name.lower().endswith(IMAGE_EXTS) + + +def build_bulk_from_coco(coco: dict, file_to_image: dict[str, str], + cat_to_class: dict[str, dict]) -> dict[str, AnnotationUpsert]: + """Pure mapping: COCO annotations -> bulk-upsert payload (unit-tested).""" + cat_name = {str(c.get("id")): c.get("name", f"class_{c.get('id')}") + for c in coco.get("categories", [])} + by_image: dict[str, list] = {} + for ann in coco.get("annotations", []): + meta = next((i for i in coco.get("images", []) + if str(i.get("id")) == str(ann.get("image_id"))), None) + if meta is None: + continue + fname = _basename(meta.get("file_name", "")) + image_id = file_to_image.get(fname) + if image_id is None: + continue + label = cat_name.get(str(ann.get("category_id")), "unknown") + cls = cat_to_class.get(label.lower()) + if cls is None: + continue + shape = {"labelId": cls["id"], "labelName": cls["name"], "color": cls["color"]} + seg = ann.get("segmentation") + if isinstance(seg, list) and seg and isinstance(seg[0], list) and len(seg[0]) >= 6: + ring = seg[0] + pts = [{"x": int(ring[i]), "y": int(ring[i + 1])} + for i in range(0, len(ring) - 1, 2)] + shape.update({"type": "polygon", "points": pts}) + elif isinstance(ann.get("bbox"), list) and len(ann["bbox"]) == 4: + x, y, w, h = (int(v) for v in ann["bbox"]) + shape.update({"type": "bbox", "bbox": {"x": x, "y": y, "width": w, "height": h}}) + else: + continue # RLE/crowd without bbox: skipped + by_image.setdefault(image_id, []).append(shape) + payload: dict[str, AnnotationUpsert] = {} + for image_id, shapes in by_image.items(): + payload[image_id] = AnnotationUpsert( + flags=Flags(), + shapes=[AnnotationShape(id=f"zip-{k}", index=k + 1, visible=True, **s) + for k, s in enumerate(shapes)], + ) + return payload + + +def _job_to_dict(job: ImportJob) -> dict: + return {"id": job.id, "project_id": job.project_id, "kind": job.kind, + "status": job.status, "source": job.source, + "total_files": job.total_files, "done_files": job.done_files, + "saved_annotations": job.saved_annotations, "error": job.error} + + +async def list_jobs(project_id: str, db: AsyncSession) -> list[dict]: + rows = (await db.execute(select(ImportJob).where( + ImportJob.project_id == project_id).order_by(ImportJob.created_at.desc())) + ).scalars().all() + return [_job_to_dict(j) for j in rows] + + +async def get_job(project_id: str, job_id: str, db: AsyncSession) -> dict: + job = await db.get(ImportJob, job_id) + if job is None or job.project_id != project_id: + raise HTTPException(404, "import job not found") + return _job_to_dict(job) + + +async def start_local_zip(project_id: str, path: str, db: AsyncSession) -> dict: + p = await db.get(Project, project_id) + if p is None: + raise HTTPException(404, "project not found") + if not os.path.isabs(path) or not path.endswith(".zip") or not os.path.exists(path): + raise HTTPException(400, "path must be an existing absolute .zip file") + job = ImportJob(project_id=project_id, kind="zip-coco", status="queued", source=path) + db.add(job) + await db.commit() + await db.refresh(job) + asyncio.create_task(_run_zip_coco(job.id, project_id, path)) + return _job_to_dict(job) + + +async def start_r2_zip(project_id: str, r2_key: str, db: AsyncSession) -> dict: + """Import a zip already in R2 (e.g. multipart-uploaded from the browser): + stream it to local temp (bounded RAM), run the standard pipeline, delete.""" + p = await db.get(Project, project_id) + if p is None: + raise HTTPException(404, "project not found") + if not r2_key.endswith(".zip"): + raise HTTPException(400, "r2_key must point to a .zip object") + job = ImportJob(project_id=project_id, kind="zip-coco", + status="queued", source=f"r2:{r2_key}") + db.add(job) + await db.commit() + await db.refresh(job) + asyncio.create_task(_run_r2_zip(job.id, project_id, r2_key)) + return _job_to_dict(job) + + +async def _run_r2_zip(job_id: str, project_id: str, r2_key: str) -> None: + import tempfile + tmp = tempfile.mktemp(prefix=f"danable-import-{job_id}-", suffix=".zip") + try: + await asyncio.to_thread(r2.download_to_file, r2_key, tmp) + await _run_zip_coco(job_id, project_id, tmp) + finally: + try: + os.unlink(tmp) + except Exception: + pass + + +async def _run_zip_coco(job_id: str, project_id: str, path: str) -> None: + t_all = time.perf_counter() + factory = get_session_factory() + async with factory() as db: + job = await db.get(ImportJob, job_id) + if job is None: + return + try: + job.status = "running" + await db.commit() + project = await db.get(Project, project_id) + if project is None: + raise RuntimeError("project gone") + + zf = zipfile.ZipFile(path) + names = zf.namelist() + coco_name = next((n for n in names if _basename(n) == "annotations.json"), + next((n for n in names if n.endswith(".json")), None)) + if coco_name is None: + raise RuntimeError("no COCO json found in zip") + coco = json.loads(zf.read(coco_name)) + + # 1. ensure label classes exist + palette = ["#3B82F6", "#10B981", "#F59E0B", "#A855F7", "#EF4444", "#06B6D4"] + existing = (await db.execute(select(LabelClass).where( + LabelClass.project_id == project_id))).scalars().all() + have = {c.name.lower() for c in existing} + for i, c in enumerate(coco.get("categories", [])): + if c.get("name", "").lower() not in have: + db.add(LabelClass(project_id=project_id, name=c["name"], + color=palette[(len(existing) + i) % len(palette)])) + have.add(c.get("name", "").lower()) + await db.commit() + + # 2. stream images: sequential reads, concurrent puts, batched commits + present = set((await db.execute(select(Image.file_name).where( + Image.project_id == project_id))).scalars().all()) + entries = [n for n in names if _is_image(n) and _basename(n) not in present] + job.total_files = len(entries) + await db.commit() + log.info("import %s: %d new files of %d", job_id, len(entries), len(names)) + + dims = {_basename(i.get("file_name", "")): + (i.get("width", 0), i.get("height", 0)) + for i in coco.get("images", [])} + sem = asyncio.Semaphore(PUT_CONCURRENCY) + + async def process_batch(batch: list[tuple[str, bytes]]) -> None: + # Concurrent network puts (bounded); then one DB commit. + results = await asyncio.gather(*[ + put_all( + keys.image_key(project.slug, f"zip-{uuid.uuid4().hex[:8]}", fname), + keys.thumbnail_key(project.slug, fname.rsplit(".", 1)[0]), + keys.thumbnail_small_key(project.slug, fname.rsplit(".", 1)[0]), + data, + ) + for fname, data in batch + ]) + rows = [] + for (fname, data), keys3 in zip(batch, results): + r2_key, thumb_key, small_key = keys3 + w, h = dims.get(fname, (0, 0)) + rows.append(Image( + project_id=project_id, file_name=fname, r2_key=r2_key, + thumbnail_r2_key=thumb_key, thumbnail_small_r2_key=small_key, + width=w, height=h, status="unassigned", flags={})) + db.add_all(rows) + await db.commit() + job.done_files = (job.done_files or 0) + len(rows) + await db.commit() + + async def put_all(r2_key: str, thumb_key: str, small_key: str, + data: bytes) -> tuple[str, str, str]: + async with sem: + await asyncio.to_thread(r2.put_bytes, r2_key, data, "image/jpeg") + try: + big = await asyncio.to_thread(resize_to_thumbnail, data, 512, 70) + tiny = await asyncio.to_thread(resize_to_thumbnail, data, 128, 65) + await asyncio.to_thread( + r2.put_bytes, thumb_key, big, "image/jpeg", IMMUTABLE) + await asyncio.to_thread( + r2.put_bytes, small_key, tiny, "image/jpeg", IMMUTABLE) + except Exception as e: + log.warning("thumb inline failed: %s", e) + return (r2_key, thumb_key, small_key) + + batch: list[tuple[str, str]] = [] + + async def flush_names() -> None: + if not batch: + return + # Sequential disk reads (zipfile is not thread-safe), then + # concurrent network puts for the whole batch at once. + loaded = [(fname, zf.read(name)) for name, fname in batch] + await process_batch(loaded) + batch.clear() + + for name in entries: + batch.append((name, _basename(name))) + if len(batch) >= 48: + await flush_names() + await flush_names() + + # 3. annotations via the bulk path in 500-image chunks + rows = (await db.execute(select(Image.id, Image.file_name).where( + Image.project_id == project_id))).all() + file_to_image = {fname: iid for iid, fname in rows} + classes = (await db.execute(select(LabelClass).where( + LabelClass.project_id == project_id))).scalars().all() + cat_to_class = {c.name.lower(): {"id": c.id, "name": c.name, "color": c.color} + for c in classes} + from src.services import annotations as ann_svc + payload = build_bulk_from_coco(coco, file_to_image, cat_to_class) + saved = 0 + chunk: dict[str, AnnotationUpsert] = {} + for iid, body in payload.items(): + chunk[iid] = body + if len(chunk) >= 500: + out = await ann_svc.bulk_upsert(project_id, chunk, db) + saved += out["saved"] + chunk = {} + if chunk: + out = await ann_svc.bulk_upsert(project_id, chunk, db) + saved += out["saved"] + job.saved_annotations = saved + job.status = "done" + await db.commit() + await events.emit("dataset.import", project_id=project_id, + payload={"via": "zip", "files": job.done_files, "saved": saved}, + latency_ms=(time.perf_counter() - t_all) * 1000) + log.info("import %s done: %d files, %d annotations", job_id, job.done_files, saved) + except Exception as e: + log.warning("import %s failed: %s", job_id, e) + try: + job = await db.get(ImportJob, job_id) + if job is not None: + job.status = "error" + job.error = str(e)[:2000] + await db.commit() + except Exception: + pass diff --git a/src/services/projects.py b/src/services/projects.py index c935675..1c7f553 100644 --- a/src/services/projects.py +++ b/src/services/projects.py @@ -44,16 +44,23 @@ async def create_project(body: ProjectCreate, db: AsyncSession) -> dict: async def _stats(project_id: str, db: AsyncSession) -> dict: - counts: dict[str, int] = {} - for st in ("unassigned", "in_progress", "pending_review", "completed"): - counts[st] = (await db.execute( - select(func.count()).where(Image.project_id == project_id, Image.status == st))).scalar() or 0 + """One GROUP BY instead of five counts + a sum: the list endpoint calls + this per project, so roundtrips dominate at 10+ projects.""" + rows = (await db.execute( + select(Image.status, func.count(), + func.coalesce(func.sum(Image.object_count), 0)) + .where(Image.project_id == project_id) + .group_by(Image.status))).all() + counts = {st: 0 for st in ("unassigned", "in_progress", "pending_review", "completed")} + objects = 0 + for st, n, o in rows: + if st in counts: + counts[st] = n or 0 + objects += int(o or 0) total = sum(counts.values()) - objects = (await db.execute( - select(func.coalesce(func.sum(Image.object_count), 0)).where(Image.project_id == project_id))).scalar() or 0 return {"totalImages": total, "unassigned": counts["unassigned"], "inProgress": counts["in_progress"], "pendingReview": counts["pending_review"], "completed": counts["completed"], - "totalObjects": int(objects)} + "totalObjects": objects} async def _members(project_id: str, db: AsyncSession) -> list[dict]: @@ -94,13 +101,28 @@ async def _thumbnail_url(project_id: str, db: AsyncSession) -> str: return "" +async def _class_counts(project_id: str, db: AsyncSession) -> dict[str, int]: + """Per-label object totals straight from stored shapes (always exact, + including bulk imports and deletions — no counter drift possible).""" + from sqlalchemy import text + rows = (await db.execute(text( + "SELECT elem->>'labelId' AS lid, COUNT(*) FROM annotations, " + "LATERAL json_array_elements(CASE WHEN shapes IS NULL THEN '[]' " + "WHEN json_typeof(shapes) = 'array' THEN shapes ELSE '[]' END) AS elem " + "WHERE project_id = :pid GROUP BY lid"), + {"pid": project_id})).all() + return {lid: n for lid, n in rows if lid} + + async def _project_card(p: Project, db: AsyncSession) -> dict: classes = (await db.execute(select(LabelClass).where(LabelClass.project_id == p.id))).scalars().all() + counts = await _class_counts(p.id, db) return {"id": p.id, "slug": p.slug, "name": p.name, "taskType": p.task_type, "status": p.status, "description": p.description, "thumbnailUrl": await _thumbnail_url(p.id, db), "updatedAt": p.updated_at.isoformat() if p.updated_at else "", - "classes": [{"id": c.id, "name": c.name, "color": c.color} for c in classes], + "classes": [{"id": c.id, "name": c.name, "color": c.color, + "count": counts.get(c.id, 0)} for c in classes], "stats": await _stats(p.id, db), "members": await _members(p.id, db)} diff --git a/src/services/users.py b/src/services/users.py index a5a6f7f..405fa0b 100644 --- a/src/services/users.py +++ b/src/services/users.py @@ -41,6 +41,17 @@ async def invite_user(email: str, role: str, project_ids: list[str], db: AsyncSe return to_public(u) +async def delete_user(user_id: str, actor_id: str | None, db: AsyncSession) -> dict: + u = await db.get(User, user_id) + if u is None: + raise HTTPException(404, "user not found") + if actor_id is not None and u.id == actor_id: + raise HTTPException(400, "cannot delete your own account") + await db.delete(u) + await db.commit() + return {"deleted": user_id} + + async def patch_user(user_id: str, patch: dict, db: AsyncSession) -> dict: u = await db.get(User, user_id) if u is None: diff --git a/tests/test_bulk.py b/tests/test_bulk.py new file mode 100644 index 0000000..eb7bb96 --- /dev/null +++ b/tests/test_bulk.py @@ -0,0 +1,84 @@ +"""Batch upload prep + bulk annotation import (COCO/YOLO path).""" +import pytest +import pytest_asyncio +from sqlalchemy import text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from shared.schemas import AnnotationUpsert +from src.services import annotations as ann_svc +from src.services import images as img_svc + + +@pytest_asyncio.fixture +async def db(): + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + from src.core.db import Base + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield async_sessionmaker(engine, expire_on_commit=False) + await engine.dispose() + + +async def _seed(db): + async with db() as s: + await s.execute(text( + "INSERT INTO projects (id, slug, name, task_type, status) " + "VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')")) + await s.execute(text( + "INSERT INTO images (id, project_id, file_name, r2_key, status, flags) " + "VALUES ('img1', 'p1', 'a.jpg', 'k1', 'in_progress', '{}')," + "('img2', 'p1', 'b.jpg', 'k2', 'in_progress', '{}')")) + await s.commit() + + +def _body(n=1): + from shared.schemas import AnnotationShape + return AnnotationUpsert(shapes=[ + AnnotationShape(id=f"s{i}", index=i + 1, type="bbox", labelId="car", + bbox={"x": 1, "y": 2, "width": 10, "height": 10}) + for i in range(n)]) + + +@pytest.mark.asyncio +async def test_bulk_upsert_saves_and_reports_missing(db): + await _seed(db) + async with db() as s: + out = await ann_svc.bulk_upsert("p1", {"img1": _body(2), "ghost": _body(1)}, s) + assert out == {"saved": 1, "missing": ["ghost"]} + n = (await s.execute(text("SELECT object_count FROM images WHERE id='img1'"))).scalar() + assert n == 2 + got = await ann_svc.get_annotation("p1", "img1", s) + assert len(got["shapes"]) == 2 + st = (await s.execute(text("SELECT status FROM images WHERE id='img1'"))).scalar() + assert st == "pending_review" + + +@pytest.mark.asyncio +async def test_bulk_upsert_unknown_project_404(db): + from fastapi import HTTPException + await _seed(db) + async with db() as s: + with pytest.raises(HTTPException) as e: + await ann_svc.bulk_upsert("nope", {"img1": _body()}, s) + assert e.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_batch_presign_unique_keys_and_cap(db): + await _seed(db) + async with db() as s: + files = [{"fileName": "same.jpg", "contentType": "image/jpeg"} for _ in range(3)] + out = await img_svc.batch_presign("p1", files, s) + keys = [o["r2Key"] for o in out] + assert len(set(keys)) == 3 and all("same.jpg" in k for k in keys) + assert all(o["putUrl"] for o in out) + + +@pytest.mark.asyncio +async def test_batch_presign_unknown_project_404(db): + from fastapi import HTTPException + await _seed(db) + async with db() as s: + with pytest.raises(HTTPException) as e: + await img_svc.batch_presign("nope", [{"fileName": "a.jpg"}], s) + assert e.value.status_code == 404 diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 0000000..8a965e6 --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,59 @@ +"""Server zip import: pure COCO mapping + job validation (no R2 touched).""" +import pytest +import pytest_asyncio +from sqlalchemy import text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from src.services.imports import build_bulk_from_coco +from src.services import imports as svc + + +@pytest_asyncio.fixture +async def db(): + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + from src.core.db import Base + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield async_sessionmaker(engine, expire_on_commit=False) + await engine.dispose() + + +COCO = { + "images": [{"id": 1, "file_name": "a.jpg"}, {"id": 2, "file_name": "b.jpg"}], + "annotations": [ + {"image_id": 1, "category_id": 1, "bbox": [10, 20, 30, 40]}, + {"image_id": 1, "category_id": 2, + "segmentation": [[0, 0, 10, 0, 10, 10, 0, 10]]}, + {"image_id": 2, "category_id": 9, "iscrowd": 1, + "segmentation": {"counts": "xyz"}}, + {"image_id": 999, "category_id": 1, "bbox": [0, 0, 5, 5]}, + ], + "categories": [{"id": 1, "name": "car"}, {"id": 2, "name": "bus"}], +} +CATS = {"car": {"id": "c1", "name": "car", "color": "#fff"}, + "bus": {"id": "c2", "name": "bus", "color": "#000"}} + + +def test_bulk_mapping_bbox_polygon_and_skips(): + payload = build_bulk_from_coco( + COCO, {"a.jpg": "img1", "b.jpg": "img2"}, CATS) + assert set(payload) == {"img1"} # b.jpg: RLE-only + unknown image 999 skipped + kinds = sorted(s.type for s in payload["img1"].shapes) + assert kinds == ["bbox", "polygon"] + poly = next(s for s in payload["img1"].shapes if s.type == "polygon") + assert [(p.x, p.y) for p in poly.points or []] == [(0, 0), (10, 0), (10, 10), (0, 10)] + assert payload["img1"].shapes[0].labelId == "c1" + + +@pytest.mark.asyncio +async def test_start_local_zip_validates(db): + from fastapi import HTTPException + async with db() as s: + await s.execute(text( + "INSERT INTO projects (id, slug, name, task_type, status) " + "VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')")) + await s.commit() + with pytest.raises(HTTPException): + await svc.start_local_zip("p1", "relative.zip", s) + with pytest.raises(HTTPException): + await svc.start_local_zip("nope", "/tmp/x.zip", s) diff --git a/tests/test_pagination.py b/tests/test_pagination.py index a5ee318..469da92 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -54,6 +54,26 @@ async def test_status_and_search_filter_server_side(db): assert one["total"] == 1 and one["items"][0]["id"] == "img1" +@pytest.mark.asyncio +async def test_locate_image_offset_matches_pages(db): + await _seed(db, 5) + async with db() as s: + loc = await svc.locate_image("p1", "img2", s) + assert loc["total"] == 5 + page = await svc.list_images("p1", "all", s, limit=2, offset=loc["offset"]) + assert page["items"][0]["id"] == "img2" + + +@pytest.mark.asyncio +async def test_locate_missing_image_is_404(db): + from fastapi import HTTPException + await _seed(db, 1) + async with db() as s: + with pytest.raises(HTTPException) as e: + await svc.locate_image("p1", "nope", s) + assert e.value.status_code == 404 + + @pytest.mark.asyncio async def test_limit_is_clamped(db): await _seed(db, 2)