mirror of
https://github.com/danagble/backend.git
synced 2026-11-16 03:00:21 +00:00
feat: add resumable dataset uploads
This commit is contained in:
parent
f8ed08b854
commit
8d40cff227
23 changed files with 913 additions and 21 deletions
|
|
@ -15,6 +15,8 @@ database:
|
|||
|
||||
storage:
|
||||
bucket_default: danagble
|
||||
# Reference quota for the sidebar meter (R2 free tier = 10 GiB).
|
||||
quota_bytes: 10737418240
|
||||
# R2 key layout. {project} = project slug, {image} = image id.
|
||||
layout:
|
||||
image: "datasets/{project}/{image}-{filename}"
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ database:
|
|||
|
||||
storage:
|
||||
bucket_default: danagble
|
||||
# Reference quota for the sidebar meter (R2 free tier = 10 GiB).
|
||||
quota_bytes: 10737418240
|
||||
# R2 key layout. {project} = project slug, {image} = image id.
|
||||
layout:
|
||||
image: "datasets/{project}/{image}-{filename}"
|
||||
|
|
|
|||
43
docs/API.md
43
docs/API.md
|
|
@ -24,6 +24,7 @@ Interactive docs: `http://localhost:8000/docs`.
|
|||
|---|---|---|
|
||||
| GET | `/health` | Liveness incl. DB check, `{ ok: true }` |
|
||||
| GET | `/metrics` | Phase-1 traffic counters per route |
|
||||
| GET | `/storage/usage` | Real bucket `{ bytes, objects, quota_bytes }` (10-min cache) |
|
||||
| GET | `/` (root, no prefix) | Name, docs and health links |
|
||||
| POST | `/admin/init-db` | Create tables, seed dev admin. Dev-only |
|
||||
| POST | `/admin/backfill-thumbnails?limit=100` | Mint missing thumbs. Dev-only |
|
||||
|
|
@ -97,6 +98,44 @@ GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0
|
|||
|
||||
`PATCH /projects/{id}/images/{imageId}?status=completed` — status transition.
|
||||
|
||||
`POST /projects/{id}/images/batch-presign` — one call for ≤200 files:
|
||||
`[{ fileName, contentType }]` → `[{ fileName, putUrl, r2Key }]`
|
||||
(keys carry unique tmp ids, so duplicate filenames never collide).
|
||||
|
||||
`GET /projects/{id}/images/by-id/{imageId}` — `{ offset, total }` in list
|
||||
order (deterministic `created_at DESC, id DESC`); backs `?imageId=` deep
|
||||
links into thousand-image datasets.
|
||||
|
||||
## Resumable multipart uploads (files ~100MB and up)
|
||||
|
||||
Small files keep the single-PUT presign path. Big single files
|
||||
(GB zips, TIFF, video) go multipart, straight to R2:
|
||||
|
||||
1. `POST /projects/{id}/uploads/multipart` `{ fileName, contentType }` →
|
||||
`{ key, uploadId, partSize: 8388608 }`.
|
||||
2. `POST .../uploads/multipart/parts` `{ key, uploadId, numbers: [1..N] }`
|
||||
→ `{ urls: { "1": "...", ... } }` (1h TTL each).
|
||||
3. PUT every part (parallel, per-part retry; store ETags client-side —
|
||||
the upload resumes after network drops instead of restarting).
|
||||
4. `POST .../uploads/multipart/complete` `{ key, uploadId,
|
||||
parts: [{ ETag, PartNumber }] }` → `{ key, etag }`.
|
||||
5. `POST .../uploads/multipart/abort` to cancel and free the upload.
|
||||
|
||||
ETag response headers are exposed via the bucket CORS policy, which the
|
||||
frontend needs to read them.
|
||||
|
||||
## Server-side dataset imports (GB-scale zips)
|
||||
|
||||
`POST /projects/{id}/imports/local-zip` `{ path }` (absolute server path)
|
||||
or `POST /projects/{id}/imports/r2-zip` `{ r2_key }` (zip already uploaded,
|
||||
e.g. via multipart) → `202 { id, status: queued, ... }`. The worker streams
|
||||
the archive (one image in RAM at a time), uploads originals + 512/128px
|
||||
thumbs concurrently, commits DB rows in batches, then bulk-upserts COCO
|
||||
annotations. Re-running skips files already present.
|
||||
|
||||
`GET /projects/{id}/imports` and `GET /projects/{id}/imports/{jobId}` poll
|
||||
`{ status, total_files, done_files, saved_annotations, error }`.
|
||||
|
||||
## Annotations (auth)
|
||||
|
||||
`GET /projects/{pid}/images/{iid}/annotation` — one row per image:
|
||||
|
|
@ -109,6 +148,10 @@ GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0
|
|||
"visible": true, "bbox": { "x": 10, "y": 20, "width": 100, "height": 80 } }] }
|
||||
```
|
||||
|
||||
`PUT /projects/{id}/annotations/batch` with `{ [imageId]: { classificationLabelId?, flags?, shapes[] } }`
|
||||
upserts up to 500 images in one commit (COCO/YOLO import path) →
|
||||
`{ saved, missing: [...] }`.
|
||||
|
||||
`PUT` same path with `{ classificationLabelId?, flags?, shapes[] }` upserts
|
||||
(one row per image, 404 when the image does not exist) and syncs
|
||||
`object_count` / flags onto the image row. Also emits an `annotation.save`
|
||||
|
|
|
|||
|
|
@ -5,10 +5,11 @@ from models.tables import (
|
|||
Annotation,
|
||||
ExportJob,
|
||||
Image,
|
||||
ImportJob,
|
||||
LabelClass,
|
||||
Project,
|
||||
User,
|
||||
)
|
||||
|
||||
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
|
||||
"LabelClass", "Project", "User"]
|
||||
"ImportJob", "LabelClass", "Project", "User"]
|
||||
|
|
|
|||
|
|
@ -104,6 +104,22 @@ class ActionEvent(Base):
|
|||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class ImportJob(Base):
|
||||
"""Server-side dataset import (e.g. 3GB zip): polled by the UI/admin."""
|
||||
__tablename__ = "imports"
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||
project_id: Mapped[str] = mapped_column(String(32), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), default="zip-coco")
|
||||
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
|
||||
source: Mapped[str] = mapped_column(String(1024), default="")
|
||||
total_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||
done_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||
saved_annotations: Mapped[int] = mapped_column(Integer, default=0)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class ExportJob(Base):
|
||||
__tablename__ = "exports"
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||
|
|
|
|||
61
shared/r2.py
61
shared/r2.py
|
|
@ -64,6 +64,67 @@ def get_bytes(key: str) -> bytes:
|
|||
return obj["Body"].read()
|
||||
|
||||
|
||||
def bucket_usage(prefix: str = "") -> dict:
|
||||
"""Sum bytes + object count via paginated LIST (no download)."""
|
||||
s = get_settings()
|
||||
total_bytes, total_objects = 0, 0
|
||||
paginator = _client().get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=s.r2_bucket, Prefix=prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
total_bytes += obj.get("Size", 0)
|
||||
total_objects += 1
|
||||
return {"bytes": total_bytes, "objects": total_objects}
|
||||
|
||||
|
||||
def create_multipart(key: str, content_type: str = "application/octet-stream") -> str:
|
||||
"""Start a multipart upload; returns upload_id. Parts upload in parallel,
|
||||
each retriable; the upload survives network drops via the stored id."""
|
||||
s = get_settings()
|
||||
res = _client().create_multipart_upload(
|
||||
Bucket=s.r2_bucket, Key=key, ContentType=content_type)
|
||||
return res["UploadId"]
|
||||
|
||||
|
||||
def presign_part(key: str, upload_id: str, part_number: int,
|
||||
expires_in: int = 3600) -> str:
|
||||
s = get_settings()
|
||||
return _client().generate_presigned_url(
|
||||
"upload_part",
|
||||
Params={"Bucket": s.r2_bucket, "Key": key,
|
||||
"UploadId": upload_id, "PartNumber": part_number},
|
||||
ExpiresIn=expires_in)
|
||||
|
||||
|
||||
def complete_multipart(key: str, upload_id: str, parts: list[dict]) -> str:
|
||||
"""parts: [{ETag, PartNumber}]. Returns final ETag."""
|
||||
s = get_settings()
|
||||
res = _client().complete_multipart_upload(
|
||||
Bucket=s.r2_bucket, Key=key, UploadId=upload_id,
|
||||
MultipartUpload={"Parts": [{"ETag": p["ETag"], "PartNumber": p["PartNumber"]}
|
||||
for p in sorted(parts, key=lambda x: x["PartNumber"])]})
|
||||
return res.get("ETag", "")
|
||||
|
||||
|
||||
def abort_multipart(key: str, upload_id: str) -> None:
|
||||
s = get_settings()
|
||||
try:
|
||||
_client().abort_multipart_upload(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
|
||||
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
|
||||
s = get_settings()
|
||||
body = _client().get_object(Bucket=s.r2_bucket, Key=key)["Body"]
|
||||
total = 0
|
||||
with open(dest_path, "wb") as f:
|
||||
for chunk in body.iter_chunks(chunk_size=chunk_bytes):
|
||||
f.write(chunk)
|
||||
total += len(chunk)
|
||||
return total
|
||||
|
||||
|
||||
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
|
||||
cache_control: str | None = None) -> None:
|
||||
s = get_settings()
|
||||
|
|
|
|||
|
|
@ -73,3 +73,14 @@ class ImageConfirm(BaseModel):
|
|||
class PresignRequest(BaseModel):
|
||||
fileName: str
|
||||
contentType: str = "image/jpeg"
|
||||
|
||||
|
||||
class BatchPresignItem(BaseModel):
|
||||
fileName: str
|
||||
contentType: str = "image/jpeg"
|
||||
|
||||
|
||||
class BulkAnnotationItem(BaseModel):
|
||||
classificationLabelId: str | None = None
|
||||
flags: Flags = Field(default_factory=Flags)
|
||||
shapes: list[AnnotationShape] = Field(default_factory=list)
|
||||
|
|
|
|||
|
|
@ -71,3 +71,11 @@ async def invite(body: InviteIn, db: AsyncSession = Depends(get_session)):
|
|||
async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)):
|
||||
from src.services import users as svc
|
||||
return await svc.patch_user(user_id, body.model_dump(), db)
|
||||
|
||||
|
||||
@users_router.delete("/{user_id}")
|
||||
async def delete_user(user_id: str, db: AsyncSession = Depends(get_session),
|
||||
user = Depends(get_current_user)):
|
||||
from src.services import users as svc
|
||||
actor = user.id if user is not None else None
|
||||
return await svc.delete_user(user_id, actor, db)
|
||||
|
|
|
|||
|
|
@ -3,13 +3,21 @@ from __future__ import annotations
|
|||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from shared.schemas import ImageConfirm, PresignRequest
|
||||
from shared.schemas import BatchPresignItem, ImageConfirm, PresignRequest
|
||||
from src.core.db import get_session
|
||||
from src.services import images as svc
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"])
|
||||
|
||||
|
||||
@router.get("/by-id/{image_id}")
|
||||
async def locate_image(project_id: str, image_id: str,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Deep-link support (?imageId=): offset of one image in list order."""
|
||||
from src.services import images as svc
|
||||
return await svc.locate_image(project_id, image_id, db)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_images(project_id: str, status: str | None = Query(default=None),
|
||||
search: str | None = Query(default=None),
|
||||
|
|
@ -26,6 +34,13 @@ async def presign_upload(project_id: str, body: PresignRequest):
|
|||
return svc.build_presign(project_id, body.fileName, body.contentType)
|
||||
|
||||
|
||||
@router.post("/batch-presign")
|
||||
async def batch_presign_upload(project_id: str, body: list[BatchPresignItem],
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""One call for a whole upload batch (≤200 files)."""
|
||||
return await svc.batch_presign(project_id, [b.model_dump() for b in body], db)
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.confirm_image(project_id, body, db)
|
||||
|
|
|
|||
42
src/api/v1/imports.py
Normal file
42
src/api/v1/imports.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.core.db import get_session
|
||||
from src.services import imports as svc
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/imports", tags=["imports"])
|
||||
|
||||
|
||||
class LocalZipIn(BaseModel):
|
||||
path: str
|
||||
|
||||
|
||||
class R2ZipIn(BaseModel):
|
||||
r2_key: str
|
||||
|
||||
|
||||
@router.post("/local-zip", status_code=202)
|
||||
async def start_local_zip(project_id: str, body: LocalZipIn,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Queue a server-side zip import (GB-scale, streamed). Poll the job id."""
|
||||
return await svc.start_local_zip(project_id, body.path, db)
|
||||
|
||||
|
||||
@router.post("/r2-zip", status_code=202)
|
||||
async def start_r2_zip(project_id: str, body: R2ZipIn,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""Queue import of a zip already uploaded to R2 (e.g. via multipart)."""
|
||||
return await svc.start_r2_zip(project_id, body.r2_key, db)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_imports(project_id: str, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.list_jobs(project_id, db)
|
||||
|
||||
|
||||
@router.get("/{job_id}")
|
||||
async def get_import(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.get_job(project_id, job_id, db)
|
||||
|
|
@ -3,8 +3,9 @@ from __future__ import annotations
|
|||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from shared.schemas import LabelClassIn, ProjectCreate
|
||||
from shared.schemas import AnnotationUpsert, BulkAnnotationItem, LabelClassIn, ProjectCreate
|
||||
from src.core.db import get_session
|
||||
from src.services import annotations as annotations_svc
|
||||
from src.services import projects as svc
|
||||
|
||||
router = APIRouter(prefix="/projects", tags=["projects"])
|
||||
|
|
@ -35,6 +36,14 @@ async def delete_project(project_id: str, db: AsyncSession = Depends(get_session
|
|||
return await svc.delete_project(project_id, db)
|
||||
|
||||
|
||||
@router.put("/{project_id}/annotations/batch")
|
||||
async def bulk_upsert_annotations(project_id: str, body: dict[str, BulkAnnotationItem],
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
"""COCO/YOLO import: upsert up to 500 images in one commit."""
|
||||
items = {image_id: AnnotationUpsert(**item.model_dump()) for image_id, item in body.items()}
|
||||
return await annotations_svc.bulk_upsert(project_id, items, db)
|
||||
|
||||
|
||||
@router.post("/{project_id}/classes", status_code=201)
|
||||
async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)):
|
||||
return await svc.add_label_class(project_id, body, db)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
|
@ -22,6 +22,28 @@ async def metrics():
|
|||
return get_snapshot()
|
||||
|
||||
|
||||
_usage_cache: dict = {"at": 0.0, "data": None}
|
||||
|
||||
|
||||
@router.get("/storage/usage")
|
||||
async def storage_usage():
|
||||
"""Real bucket bytes/objects for the sidebar meter (10-min cache)."""
|
||||
import time
|
||||
from shared import r2
|
||||
from src.core.config import get_settings
|
||||
now = time.time()
|
||||
if _usage_cache["data"] is not None and now - _usage_cache["at"] < 600:
|
||||
return _usage_cache["data"]
|
||||
try:
|
||||
usage = r2.bucket_usage()
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
data = {"bytes": usage["bytes"], "objects": usage["objects"],
|
||||
"quota_bytes": get_settings().storage_quota_bytes}
|
||||
_usage_cache.update({"at": now, "data": data})
|
||||
return data
|
||||
|
||||
|
||||
@router.post("/admin/backfill-thumbnails")
|
||||
async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)):
|
||||
"""Mint thumbs for images uploaded before thumbnailing existed. Dev-only."""
|
||||
|
|
|
|||
102
src/api/v1/uploads.py
Normal file
102
src/api/v1/uploads.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
"""Resumable multipart uploads: big single files (GB zip/TIFF/video).
|
||||
|
||||
Flow: POST /multipart -> {key, uploadId} -> PUT each part to its presigned
|
||||
URL (parallel, retried per part, resumable via stored ETags) ->
|
||||
POST /multipart/complete. Use for anything over ~100MB; small files keep
|
||||
the single-PUT presign path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Project
|
||||
from shared import r2
|
||||
from src.core.db import get_session
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/uploads", tags=["uploads"])
|
||||
|
||||
PART_SIZE = 8 * 1024 * 1024
|
||||
|
||||
|
||||
class MultipartStart(BaseModel):
|
||||
fileName: str
|
||||
contentType: str = "application/octet-stream"
|
||||
|
||||
|
||||
class MultipartParts(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
numbers: list[int] = Field(max_length=1000)
|
||||
|
||||
|
||||
class PartETag(BaseModel):
|
||||
ETag: str
|
||||
PartNumber: int
|
||||
|
||||
|
||||
class MultipartComplete(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
parts: list[PartETag]
|
||||
|
||||
|
||||
class MultipartAbort(BaseModel):
|
||||
key: str
|
||||
uploadId: str
|
||||
|
||||
|
||||
async def _project_slug(project_id: str, db: AsyncSession) -> str:
|
||||
p = await db.get(Project, project_id)
|
||||
if p is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
return p.slug
|
||||
|
||||
|
||||
@router.post("/multipart")
|
||||
async def start_multipart(project_id: str, body: MultipartStart,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
import uuid as _uuid
|
||||
slug = await _project_slug(project_id, db)
|
||||
key = f"uploads/{slug}/{_uuid.uuid4().hex[:12]}-{body.fileName.replace('/', '_')}"
|
||||
try:
|
||||
upload_id = r2.create_multipart(key, body.contentType)
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
|
||||
|
||||
|
||||
@router.post("/multipart/parts")
|
||||
async def part_urls(project_id: str, body: MultipartParts,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
if any(n < 1 or n > 10000 for n in body.numbers):
|
||||
raise HTTPException(400, "invalid part numbers")
|
||||
try:
|
||||
urls = {n: r2.presign_part(body.key, body.uploadId, n) for n in body.numbers}
|
||||
except Exception as e:
|
||||
raise HTTPException(503, f"storage unreachable: {e}")
|
||||
return {"urls": urls}
|
||||
|
||||
|
||||
@router.post("/multipart/complete")
|
||||
async def complete_multipart(project_id: str, body: MultipartComplete,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
try:
|
||||
etag = r2.complete_multipart(body.key, body.uploadId,
|
||||
[p.model_dump() for p in body.parts])
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
|
||||
if not r2.head_exists(body.key):
|
||||
raise HTTPException(400, "object not found after complete")
|
||||
return {"key": body.key, "etag": etag}
|
||||
|
||||
|
||||
@router.post("/multipart/abort")
|
||||
async def abort_multipart(project_id: str, body: MultipartAbort,
|
||||
db: AsyncSession = Depends(get_session)):
|
||||
await _project_slug(project_id, db)
|
||||
r2.abort_multipart(body.key, body.uploadId)
|
||||
return {"aborted": True}
|
||||
|
|
@ -46,6 +46,7 @@ class Settings:
|
|||
layout = storage.get("layout", {})
|
||||
presigned = storage.get("presigned_ttl", {})
|
||||
self.r2_layout: dict = layout
|
||||
self.storage_quota_bytes: int = int(storage.get("quota_bytes", 10 * 1024**3))
|
||||
thumb = storage.get("thumbnail", {})
|
||||
self.thumbnail_enabled: bool = bool(thumb.get("enabled", True))
|
||||
self.thumbnail_max_side: int = int(thumb.get("max_side", 512))
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from shared import events
|
|||
from shared.logger import get_logger
|
||||
from shared.tracing import TraceMiddleware
|
||||
from shared.traffic import TrafficMiddleware
|
||||
from src.api.v1 import activities, annotations, auth, events as events_api, images, projects, system
|
||||
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads
|
||||
from src.api.v1.auth import users_router
|
||||
from src.core.config import get_settings
|
||||
from src.core.security import get_current_user
|
||||
|
|
@ -61,6 +61,8 @@ app.include_router(activities.router, prefix=prefix, dependencies=authed)
|
|||
app.include_router(projects.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(images.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(imports.router, prefix=prefix, dependencies=authed)
|
||||
app.include_router(uploads.router, prefix=prefix, dependencies=authed)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import time
|
|||
from fastapi import HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Annotation, Image
|
||||
from models import Annotation, Image, Project
|
||||
from shared import events
|
||||
from shared.schemas import AnnotationUpsert
|
||||
|
||||
|
|
@ -29,10 +29,8 @@ async def get_annotation(project_id: str, image_id: str, db: AsyncSession) -> di
|
|||
"flags": ann.flags, "shapes": ann.shapes}
|
||||
|
||||
|
||||
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
|
||||
db: AsyncSession) -> dict:
|
||||
im = await _require_image(project_id, image_id, db)
|
||||
t0 = time.perf_counter()
|
||||
async def _apply_upsert(db: AsyncSession, project_id: str, image_id: str,
|
||||
body: AnnotationUpsert) -> int:
|
||||
ann = await db.get(Annotation, image_id)
|
||||
payload = {"project_id": project_id,
|
||||
"classification_label_id": body.classificationLabelId,
|
||||
|
|
@ -49,6 +47,43 @@ async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpse
|
|||
im.object_count = len(body.shapes)
|
||||
im.classification_label_id = body.classificationLabelId
|
||||
im.flags = body.flags.model_dump()
|
||||
return len(body.shapes)
|
||||
|
||||
|
||||
async def bulk_upsert(project_id: str, items: dict[str, AnnotationUpsert],
|
||||
db: AsyncSession) -> dict:
|
||||
"""COCO/YOLO import path: one commit for the whole batch (≤500 images).
|
||||
|
||||
Imported labels are machine data awaiting human review, so saved rows go
|
||||
to pending_review — that is what the review queue and stats count on.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
if await db.get(Project, project_id) is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
from shared import events
|
||||
import time
|
||||
saved, missing = 0, []
|
||||
t0 = time.perf_counter()
|
||||
for image_id, body in list(items.items())[:500]:
|
||||
im = await db.get(Image, image_id)
|
||||
if im is None or im.project_id != project_id:
|
||||
missing.append(image_id)
|
||||
continue
|
||||
await _apply_upsert(db, project_id, image_id, body)
|
||||
im.status = "pending_review"
|
||||
saved += 1
|
||||
await db.commit()
|
||||
await events.emit("dataset.import", project_id=project_id,
|
||||
payload={"saved": saved, "missing": len(missing)},
|
||||
latency_ms=(time.perf_counter() - t0) * 1000)
|
||||
return {"saved": saved, "missing": missing}
|
||||
|
||||
|
||||
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
|
||||
db: AsyncSession) -> dict:
|
||||
await _require_image(project_id, image_id, db)
|
||||
t0 = time.perf_counter()
|
||||
await _apply_upsert(db, project_id, image_id, body)
|
||||
await db.commit()
|
||||
await events.emit("annotation.save", project_id=project_id, image_id=image_id,
|
||||
payload={"shapeCount": len(body.shapes),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from fastapi import HTTPException
|
|||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Image
|
||||
from models import Image, Project
|
||||
from shared import events, keys, r2
|
||||
from shared.schemas import ImageConfirm
|
||||
|
||||
|
|
@ -40,7 +40,10 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession,
|
|||
q = q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
|
||||
count_q = count_q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
|
||||
total = (await db.execute(count_q)).scalar() or 0
|
||||
rows = (await db.execute(q.order_by(Image.created_at.desc()).limit(limit).offset(offset))).scalars().all()
|
||||
# Deterministic order (id tiebreak): pages must not shift underfoot.
|
||||
rows = (await db.execute(
|
||||
q.order_by(Image.created_at.desc(), Image.id.desc()).limit(limit).offset(offset)
|
||||
)).scalars().all()
|
||||
users = (await db.execute(select(User))).scalars().all()
|
||||
names = {u.id: u.name for u in users}
|
||||
items = [{"id": im.id, "projectId": im.project_id, "fileName": im.file_name,
|
||||
|
|
@ -57,12 +60,44 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession,
|
|||
return {"items": items, "total": total}
|
||||
|
||||
|
||||
async def locate_image(project_id: str, image_id: str, db: AsyncSession) -> dict:
|
||||
"""Offset (0-based) of one image in list order, for ?imageId= deep links."""
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import func, text
|
||||
total = (await db.execute(
|
||||
select(func.count()).where(Image.project_id == project_id))).scalar() or 0
|
||||
row = (await db.execute(text(
|
||||
"SELECT rn FROM (SELECT id, ROW_NUMBER() OVER "
|
||||
"(ORDER BY created_at DESC, id DESC) AS rn FROM images "
|
||||
"WHERE project_id = :pid) t WHERE id = :iid"),
|
||||
{"pid": project_id, "iid": image_id})).first()
|
||||
if row is None:
|
||||
raise HTTPException(404, "image not found")
|
||||
return {"offset": int(row[0]) - 1, "total": total}
|
||||
|
||||
|
||||
def build_presign(project_id: str, file_name: str, content_type: str) -> dict:
|
||||
"""Step 1/2 direct upload: frontend PUTs bytes straight to R2 with this URL."""
|
||||
key = keys.image_key(project_id, "tmp", file_name) # real id assigned on confirm
|
||||
import uuid as _uuid
|
||||
# Unique tmp id per file: duplicate filenames in one batch must not collide.
|
||||
key = keys.image_key(project_id, f"tmp-{_uuid.uuid4().hex[:8]}", file_name)
|
||||
return {"putUrl": r2.presign_put(key, content_type), "r2Key": key}
|
||||
|
||||
|
||||
async def batch_presign(project_id: str, files: list[dict], db: AsyncSession) -> list[dict]:
|
||||
"""One call for a whole batch (≤200): avoids a presign roundtrip per file."""
|
||||
if await db.get(Project, project_id) is None:
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(404, "project not found")
|
||||
out = []
|
||||
for f in files[:200]:
|
||||
slot = build_presign(project_id, f.get("fileName", "upload.jpg"),
|
||||
f.get("contentType", "image/jpeg"))
|
||||
slot["fileName"] = f.get("fileName", "upload.jpg")
|
||||
out.append(slot)
|
||||
return out
|
||||
|
||||
|
||||
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession) -> dict:
|
||||
"""Step 2/2: metadata row after the R2 PUT succeeds."""
|
||||
if not r2.head_exists(body.r2Key):
|
||||
|
|
|
|||
289
src/services/imports.py
Normal file
289
src/services/imports.py
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
"""Server-side dataset import: stream a big zip (GBs) without loading it to RAM.
|
||||
|
||||
One image in memory at a time for reads; network puts run concurrently;
|
||||
DB commits in batches of 100. Re-runnable: files already in the project
|
||||
(by file_name) are skipped.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import zipfile
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models import Image, ImportJob, LabelClass, Project
|
||||
from shared import events, keys, r2
|
||||
from shared.logger import get_logger
|
||||
from shared.schemas import AnnotationShape, AnnotationUpsert, Flags
|
||||
from shared.thumbnails import IMMUTABLE, resize_to_thumbnail
|
||||
from src.core.db import get_session_factory
|
||||
|
||||
log = get_logger("danable.imports")
|
||||
|
||||
IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp", ".bmp")
|
||||
PUT_CONCURRENCY = 24
|
||||
DB_BATCH = 100
|
||||
|
||||
|
||||
def _basename(name: str) -> str:
|
||||
return name.split("/")[-1]
|
||||
|
||||
|
||||
def _is_image(name: str) -> bool:
|
||||
return name.lower().endswith(IMAGE_EXTS)
|
||||
|
||||
|
||||
def build_bulk_from_coco(coco: dict, file_to_image: dict[str, str],
|
||||
cat_to_class: dict[str, dict]) -> dict[str, AnnotationUpsert]:
|
||||
"""Pure mapping: COCO annotations -> bulk-upsert payload (unit-tested)."""
|
||||
cat_name = {str(c.get("id")): c.get("name", f"class_{c.get('id')}")
|
||||
for c in coco.get("categories", [])}
|
||||
by_image: dict[str, list] = {}
|
||||
for ann in coco.get("annotations", []):
|
||||
meta = next((i for i in coco.get("images", [])
|
||||
if str(i.get("id")) == str(ann.get("image_id"))), None)
|
||||
if meta is None:
|
||||
continue
|
||||
fname = _basename(meta.get("file_name", ""))
|
||||
image_id = file_to_image.get(fname)
|
||||
if image_id is None:
|
||||
continue
|
||||
label = cat_name.get(str(ann.get("category_id")), "unknown")
|
||||
cls = cat_to_class.get(label.lower())
|
||||
if cls is None:
|
||||
continue
|
||||
shape = {"labelId": cls["id"], "labelName": cls["name"], "color": cls["color"]}
|
||||
seg = ann.get("segmentation")
|
||||
if isinstance(seg, list) and seg and isinstance(seg[0], list) and len(seg[0]) >= 6:
|
||||
ring = seg[0]
|
||||
pts = [{"x": int(ring[i]), "y": int(ring[i + 1])}
|
||||
for i in range(0, len(ring) - 1, 2)]
|
||||
shape.update({"type": "polygon", "points": pts})
|
||||
elif isinstance(ann.get("bbox"), list) and len(ann["bbox"]) == 4:
|
||||
x, y, w, h = (int(v) for v in ann["bbox"])
|
||||
shape.update({"type": "bbox", "bbox": {"x": x, "y": y, "width": w, "height": h}})
|
||||
else:
|
||||
continue # RLE/crowd without bbox: skipped
|
||||
by_image.setdefault(image_id, []).append(shape)
|
||||
payload: dict[str, AnnotationUpsert] = {}
|
||||
for image_id, shapes in by_image.items():
|
||||
payload[image_id] = AnnotationUpsert(
|
||||
flags=Flags(),
|
||||
shapes=[AnnotationShape(id=f"zip-{k}", index=k + 1, visible=True, **s)
|
||||
for k, s in enumerate(shapes)],
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _job_to_dict(job: ImportJob) -> dict:
|
||||
return {"id": job.id, "project_id": job.project_id, "kind": job.kind,
|
||||
"status": job.status, "source": job.source,
|
||||
"total_files": job.total_files, "done_files": job.done_files,
|
||||
"saved_annotations": job.saved_annotations, "error": job.error}
|
||||
|
||||
|
||||
async def list_jobs(project_id: str, db: AsyncSession) -> list[dict]:
|
||||
rows = (await db.execute(select(ImportJob).where(
|
||||
ImportJob.project_id == project_id).order_by(ImportJob.created_at.desc()))
|
||||
).scalars().all()
|
||||
return [_job_to_dict(j) for j in rows]
|
||||
|
||||
|
||||
async def get_job(project_id: str, job_id: str, db: AsyncSession) -> dict:
|
||||
job = await db.get(ImportJob, job_id)
|
||||
if job is None or job.project_id != project_id:
|
||||
raise HTTPException(404, "import job not found")
|
||||
return _job_to_dict(job)
|
||||
|
||||
|
||||
async def start_local_zip(project_id: str, path: str, db: AsyncSession) -> dict:
|
||||
p = await db.get(Project, project_id)
|
||||
if p is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
if not os.path.isabs(path) or not path.endswith(".zip") or not os.path.exists(path):
|
||||
raise HTTPException(400, "path must be an existing absolute .zip file")
|
||||
job = ImportJob(project_id=project_id, kind="zip-coco", status="queued", source=path)
|
||||
db.add(job)
|
||||
await db.commit()
|
||||
await db.refresh(job)
|
||||
asyncio.create_task(_run_zip_coco(job.id, project_id, path))
|
||||
return _job_to_dict(job)
|
||||
|
||||
|
||||
async def start_r2_zip(project_id: str, r2_key: str, db: AsyncSession) -> dict:
|
||||
"""Import a zip already in R2 (e.g. multipart-uploaded from the browser):
|
||||
stream it to local temp (bounded RAM), run the standard pipeline, delete."""
|
||||
p = await db.get(Project, project_id)
|
||||
if p is None:
|
||||
raise HTTPException(404, "project not found")
|
||||
if not r2_key.endswith(".zip"):
|
||||
raise HTTPException(400, "r2_key must point to a .zip object")
|
||||
job = ImportJob(project_id=project_id, kind="zip-coco",
|
||||
status="queued", source=f"r2:{r2_key}")
|
||||
db.add(job)
|
||||
await db.commit()
|
||||
await db.refresh(job)
|
||||
asyncio.create_task(_run_r2_zip(job.id, project_id, r2_key))
|
||||
return _job_to_dict(job)
|
||||
|
||||
|
||||
async def _run_r2_zip(job_id: str, project_id: str, r2_key: str) -> None:
|
||||
import tempfile
|
||||
tmp = tempfile.mktemp(prefix=f"danable-import-{job_id}-", suffix=".zip")
|
||||
try:
|
||||
await asyncio.to_thread(r2.download_to_file, r2_key, tmp)
|
||||
await _run_zip_coco(job_id, project_id, tmp)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _run_zip_coco(job_id: str, project_id: str, path: str) -> None:
|
||||
t_all = time.perf_counter()
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
job = await db.get(ImportJob, job_id)
|
||||
if job is None:
|
||||
return
|
||||
try:
|
||||
job.status = "running"
|
||||
await db.commit()
|
||||
project = await db.get(Project, project_id)
|
||||
if project is None:
|
||||
raise RuntimeError("project gone")
|
||||
|
||||
zf = zipfile.ZipFile(path)
|
||||
names = zf.namelist()
|
||||
coco_name = next((n for n in names if _basename(n) == "annotations.json"),
|
||||
next((n for n in names if n.endswith(".json")), None))
|
||||
if coco_name is None:
|
||||
raise RuntimeError("no COCO json found in zip")
|
||||
coco = json.loads(zf.read(coco_name))
|
||||
|
||||
# 1. ensure label classes exist
|
||||
palette = ["#3B82F6", "#10B981", "#F59E0B", "#A855F7", "#EF4444", "#06B6D4"]
|
||||
existing = (await db.execute(select(LabelClass).where(
|
||||
LabelClass.project_id == project_id))).scalars().all()
|
||||
have = {c.name.lower() for c in existing}
|
||||
for i, c in enumerate(coco.get("categories", [])):
|
||||
if c.get("name", "").lower() not in have:
|
||||
db.add(LabelClass(project_id=project_id, name=c["name"],
|
||||
color=palette[(len(existing) + i) % len(palette)]))
|
||||
have.add(c.get("name", "").lower())
|
||||
await db.commit()
|
||||
|
||||
# 2. stream images: sequential reads, concurrent puts, batched commits
|
||||
present = set((await db.execute(select(Image.file_name).where(
|
||||
Image.project_id == project_id))).scalars().all())
|
||||
entries = [n for n in names if _is_image(n) and _basename(n) not in present]
|
||||
job.total_files = len(entries)
|
||||
await db.commit()
|
||||
log.info("import %s: %d new files of %d", job_id, len(entries), len(names))
|
||||
|
||||
dims = {_basename(i.get("file_name", "")):
|
||||
(i.get("width", 0), i.get("height", 0))
|
||||
for i in coco.get("images", [])}
|
||||
sem = asyncio.Semaphore(PUT_CONCURRENCY)
|
||||
|
||||
async def process_batch(batch: list[tuple[str, bytes]]) -> None:
|
||||
# Concurrent network puts (bounded); then one DB commit.
|
||||
results = await asyncio.gather(*[
|
||||
put_all(
|
||||
keys.image_key(project.slug, f"zip-{uuid.uuid4().hex[:8]}", fname),
|
||||
keys.thumbnail_key(project.slug, fname.rsplit(".", 1)[0]),
|
||||
keys.thumbnail_small_key(project.slug, fname.rsplit(".", 1)[0]),
|
||||
data,
|
||||
)
|
||||
for fname, data in batch
|
||||
])
|
||||
rows = []
|
||||
for (fname, data), keys3 in zip(batch, results):
|
||||
r2_key, thumb_key, small_key = keys3
|
||||
w, h = dims.get(fname, (0, 0))
|
||||
rows.append(Image(
|
||||
project_id=project_id, file_name=fname, r2_key=r2_key,
|
||||
thumbnail_r2_key=thumb_key, thumbnail_small_r2_key=small_key,
|
||||
width=w, height=h, status="unassigned", flags={}))
|
||||
db.add_all(rows)
|
||||
await db.commit()
|
||||
job.done_files = (job.done_files or 0) + len(rows)
|
||||
await db.commit()
|
||||
|
||||
async def put_all(r2_key: str, thumb_key: str, small_key: str,
|
||||
data: bytes) -> tuple[str, str, str]:
|
||||
async with sem:
|
||||
await asyncio.to_thread(r2.put_bytes, r2_key, data, "image/jpeg")
|
||||
try:
|
||||
big = await asyncio.to_thread(resize_to_thumbnail, data, 512, 70)
|
||||
tiny = await asyncio.to_thread(resize_to_thumbnail, data, 128, 65)
|
||||
await asyncio.to_thread(
|
||||
r2.put_bytes, thumb_key, big, "image/jpeg", IMMUTABLE)
|
||||
await asyncio.to_thread(
|
||||
r2.put_bytes, small_key, tiny, "image/jpeg", IMMUTABLE)
|
||||
except Exception as e:
|
||||
log.warning("thumb inline failed: %s", e)
|
||||
return (r2_key, thumb_key, small_key)
|
||||
|
||||
batch: list[tuple[str, str]] = []
|
||||
|
||||
async def flush_names() -> None:
|
||||
if not batch:
|
||||
return
|
||||
# Sequential disk reads (zipfile is not thread-safe), then
|
||||
# concurrent network puts for the whole batch at once.
|
||||
loaded = [(fname, zf.read(name)) for name, fname in batch]
|
||||
await process_batch(loaded)
|
||||
batch.clear()
|
||||
|
||||
for name in entries:
|
||||
batch.append((name, _basename(name)))
|
||||
if len(batch) >= 48:
|
||||
await flush_names()
|
||||
await flush_names()
|
||||
|
||||
# 3. annotations via the bulk path in 500-image chunks
|
||||
rows = (await db.execute(select(Image.id, Image.file_name).where(
|
||||
Image.project_id == project_id))).all()
|
||||
file_to_image = {fname: iid for iid, fname in rows}
|
||||
classes = (await db.execute(select(LabelClass).where(
|
||||
LabelClass.project_id == project_id))).scalars().all()
|
||||
cat_to_class = {c.name.lower(): {"id": c.id, "name": c.name, "color": c.color}
|
||||
for c in classes}
|
||||
from src.services import annotations as ann_svc
|
||||
payload = build_bulk_from_coco(coco, file_to_image, cat_to_class)
|
||||
saved = 0
|
||||
chunk: dict[str, AnnotationUpsert] = {}
|
||||
for iid, body in payload.items():
|
||||
chunk[iid] = body
|
||||
if len(chunk) >= 500:
|
||||
out = await ann_svc.bulk_upsert(project_id, chunk, db)
|
||||
saved += out["saved"]
|
||||
chunk = {}
|
||||
if chunk:
|
||||
out = await ann_svc.bulk_upsert(project_id, chunk, db)
|
||||
saved += out["saved"]
|
||||
job.saved_annotations = saved
|
||||
job.status = "done"
|
||||
await db.commit()
|
||||
await events.emit("dataset.import", project_id=project_id,
|
||||
payload={"via": "zip", "files": job.done_files, "saved": saved},
|
||||
latency_ms=(time.perf_counter() - t_all) * 1000)
|
||||
log.info("import %s done: %d files, %d annotations", job_id, job.done_files, saved)
|
||||
except Exception as e:
|
||||
log.warning("import %s failed: %s", job_id, e)
|
||||
try:
|
||||
job = await db.get(ImportJob, job_id)
|
||||
if job is not None:
|
||||
job.status = "error"
|
||||
job.error = str(e)[:2000]
|
||||
await db.commit()
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -44,16 +44,23 @@ async def create_project(body: ProjectCreate, db: AsyncSession) -> dict:
|
|||
|
||||
|
||||
async def _stats(project_id: str, db: AsyncSession) -> dict:
|
||||
counts: dict[str, int] = {}
|
||||
for st in ("unassigned", "in_progress", "pending_review", "completed"):
|
||||
counts[st] = (await db.execute(
|
||||
select(func.count()).where(Image.project_id == project_id, Image.status == st))).scalar() or 0
|
||||
"""One GROUP BY instead of five counts + a sum: the list endpoint calls
|
||||
this per project, so roundtrips dominate at 10+ projects."""
|
||||
rows = (await db.execute(
|
||||
select(Image.status, func.count(),
|
||||
func.coalesce(func.sum(Image.object_count), 0))
|
||||
.where(Image.project_id == project_id)
|
||||
.group_by(Image.status))).all()
|
||||
counts = {st: 0 for st in ("unassigned", "in_progress", "pending_review", "completed")}
|
||||
objects = 0
|
||||
for st, n, o in rows:
|
||||
if st in counts:
|
||||
counts[st] = n or 0
|
||||
objects += int(o or 0)
|
||||
total = sum(counts.values())
|
||||
objects = (await db.execute(
|
||||
select(func.coalesce(func.sum(Image.object_count), 0)).where(Image.project_id == project_id))).scalar() or 0
|
||||
return {"totalImages": total, "unassigned": counts["unassigned"], "inProgress": counts["in_progress"],
|
||||
"pendingReview": counts["pending_review"], "completed": counts["completed"],
|
||||
"totalObjects": int(objects)}
|
||||
"totalObjects": objects}
|
||||
|
||||
|
||||
async def _members(project_id: str, db: AsyncSession) -> list[dict]:
|
||||
|
|
@ -94,13 +101,28 @@ async def _thumbnail_url(project_id: str, db: AsyncSession) -> str:
|
|||
return ""
|
||||
|
||||
|
||||
async def _class_counts(project_id: str, db: AsyncSession) -> dict[str, int]:
|
||||
"""Per-label object totals straight from stored shapes (always exact,
|
||||
including bulk imports and deletions — no counter drift possible)."""
|
||||
from sqlalchemy import text
|
||||
rows = (await db.execute(text(
|
||||
"SELECT elem->>'labelId' AS lid, COUNT(*) FROM annotations, "
|
||||
"LATERAL json_array_elements(CASE WHEN shapes IS NULL THEN '[]' "
|
||||
"WHEN json_typeof(shapes) = 'array' THEN shapes ELSE '[]' END) AS elem "
|
||||
"WHERE project_id = :pid GROUP BY lid"),
|
||||
{"pid": project_id})).all()
|
||||
return {lid: n for lid, n in rows if lid}
|
||||
|
||||
|
||||
async def _project_card(p: Project, db: AsyncSession) -> dict:
|
||||
classes = (await db.execute(select(LabelClass).where(LabelClass.project_id == p.id))).scalars().all()
|
||||
counts = await _class_counts(p.id, db)
|
||||
return {"id": p.id, "slug": p.slug, "name": p.name, "taskType": p.task_type,
|
||||
"status": p.status, "description": p.description,
|
||||
"thumbnailUrl": await _thumbnail_url(p.id, db),
|
||||
"updatedAt": p.updated_at.isoformat() if p.updated_at else "",
|
||||
"classes": [{"id": c.id, "name": c.name, "color": c.color} for c in classes],
|
||||
"classes": [{"id": c.id, "name": c.name, "color": c.color,
|
||||
"count": counts.get(c.id, 0)} for c in classes],
|
||||
"stats": await _stats(p.id, db),
|
||||
"members": await _members(p.id, db)}
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,17 @@ async def invite_user(email: str, role: str, project_ids: list[str], db: AsyncSe
|
|||
return to_public(u)
|
||||
|
||||
|
||||
async def delete_user(user_id: str, actor_id: str | None, db: AsyncSession) -> dict:
|
||||
u = await db.get(User, user_id)
|
||||
if u is None:
|
||||
raise HTTPException(404, "user not found")
|
||||
if actor_id is not None and u.id == actor_id:
|
||||
raise HTTPException(400, "cannot delete your own account")
|
||||
await db.delete(u)
|
||||
await db.commit()
|
||||
return {"deleted": user_id}
|
||||
|
||||
|
||||
async def patch_user(user_id: str, patch: dict, db: AsyncSession) -> dict:
|
||||
u = await db.get(User, user_id)
|
||||
if u is None:
|
||||
|
|
|
|||
84
tests/test_bulk.py
Normal file
84
tests/test_bulk.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Batch upload prep + bulk annotation import (COCO/YOLO path)."""
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from shared.schemas import AnnotationUpsert
|
||||
from src.services import annotations as ann_svc
|
||||
from src.services import images as img_svc
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
from src.core.db import Base
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
yield async_sessionmaker(engine, expire_on_commit=False)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _seed(db):
|
||||
async with db() as s:
|
||||
await s.execute(text(
|
||||
"INSERT INTO projects (id, slug, name, task_type, status) "
|
||||
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
|
||||
await s.execute(text(
|
||||
"INSERT INTO images (id, project_id, file_name, r2_key, status, flags) "
|
||||
"VALUES ('img1', 'p1', 'a.jpg', 'k1', 'in_progress', '{}'),"
|
||||
"('img2', 'p1', 'b.jpg', 'k2', 'in_progress', '{}')"))
|
||||
await s.commit()
|
||||
|
||||
|
||||
def _body(n=1):
|
||||
from shared.schemas import AnnotationShape
|
||||
return AnnotationUpsert(shapes=[
|
||||
AnnotationShape(id=f"s{i}", index=i + 1, type="bbox", labelId="car",
|
||||
bbox={"x": 1, "y": 2, "width": 10, "height": 10})
|
||||
for i in range(n)])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_upsert_saves_and_reports_missing(db):
|
||||
await _seed(db)
|
||||
async with db() as s:
|
||||
out = await ann_svc.bulk_upsert("p1", {"img1": _body(2), "ghost": _body(1)}, s)
|
||||
assert out == {"saved": 1, "missing": ["ghost"]}
|
||||
n = (await s.execute(text("SELECT object_count FROM images WHERE id='img1'"))).scalar()
|
||||
assert n == 2
|
||||
got = await ann_svc.get_annotation("p1", "img1", s)
|
||||
assert len(got["shapes"]) == 2
|
||||
st = (await s.execute(text("SELECT status FROM images WHERE id='img1'"))).scalar()
|
||||
assert st == "pending_review"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_upsert_unknown_project_404(db):
|
||||
from fastapi import HTTPException
|
||||
await _seed(db)
|
||||
async with db() as s:
|
||||
with pytest.raises(HTTPException) as e:
|
||||
await ann_svc.bulk_upsert("nope", {"img1": _body()}, s)
|
||||
assert e.value.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_presign_unique_keys_and_cap(db):
|
||||
await _seed(db)
|
||||
async with db() as s:
|
||||
files = [{"fileName": "same.jpg", "contentType": "image/jpeg"} for _ in range(3)]
|
||||
out = await img_svc.batch_presign("p1", files, s)
|
||||
keys = [o["r2Key"] for o in out]
|
||||
assert len(set(keys)) == 3 and all("same.jpg" in k for k in keys)
|
||||
assert all(o["putUrl"] for o in out)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_presign_unknown_project_404(db):
|
||||
from fastapi import HTTPException
|
||||
await _seed(db)
|
||||
async with db() as s:
|
||||
with pytest.raises(HTTPException) as e:
|
||||
await img_svc.batch_presign("nope", [{"fileName": "a.jpg"}], s)
|
||||
assert e.value.status_code == 404
|
||||
59
tests/test_imports.py
Normal file
59
tests/test_imports.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""Server zip import: pure COCO mapping + job validation (no R2 touched)."""
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from src.services.imports import build_bulk_from_coco
|
||||
from src.services import imports as svc
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
from src.core.db import Base
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
yield async_sessionmaker(engine, expire_on_commit=False)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
COCO = {
|
||||
"images": [{"id": 1, "file_name": "a.jpg"}, {"id": 2, "file_name": "b.jpg"}],
|
||||
"annotations": [
|
||||
{"image_id": 1, "category_id": 1, "bbox": [10, 20, 30, 40]},
|
||||
{"image_id": 1, "category_id": 2,
|
||||
"segmentation": [[0, 0, 10, 0, 10, 10, 0, 10]]},
|
||||
{"image_id": 2, "category_id": 9, "iscrowd": 1,
|
||||
"segmentation": {"counts": "xyz"}},
|
||||
{"image_id": 999, "category_id": 1, "bbox": [0, 0, 5, 5]},
|
||||
],
|
||||
"categories": [{"id": 1, "name": "car"}, {"id": 2, "name": "bus"}],
|
||||
}
|
||||
CATS = {"car": {"id": "c1", "name": "car", "color": "#fff"},
|
||||
"bus": {"id": "c2", "name": "bus", "color": "#000"}}
|
||||
|
||||
|
||||
def test_bulk_mapping_bbox_polygon_and_skips():
|
||||
payload = build_bulk_from_coco(
|
||||
COCO, {"a.jpg": "img1", "b.jpg": "img2"}, CATS)
|
||||
assert set(payload) == {"img1"} # b.jpg: RLE-only + unknown image 999 skipped
|
||||
kinds = sorted(s.type for s in payload["img1"].shapes)
|
||||
assert kinds == ["bbox", "polygon"]
|
||||
poly = next(s for s in payload["img1"].shapes if s.type == "polygon")
|
||||
assert [(p.x, p.y) for p in poly.points or []] == [(0, 0), (10, 0), (10, 10), (0, 10)]
|
||||
assert payload["img1"].shapes[0].labelId == "c1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_local_zip_validates(db):
|
||||
from fastapi import HTTPException
|
||||
async with db() as s:
|
||||
await s.execute(text(
|
||||
"INSERT INTO projects (id, slug, name, task_type, status) "
|
||||
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
|
||||
await s.commit()
|
||||
with pytest.raises(HTTPException):
|
||||
await svc.start_local_zip("p1", "relative.zip", s)
|
||||
with pytest.raises(HTTPException):
|
||||
await svc.start_local_zip("nope", "/tmp/x.zip", s)
|
||||
|
|
@ -54,6 +54,26 @@ async def test_status_and_search_filter_server_side(db):
|
|||
assert one["total"] == 1 and one["items"][0]["id"] == "img1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locate_image_offset_matches_pages(db):
|
||||
await _seed(db, 5)
|
||||
async with db() as s:
|
||||
loc = await svc.locate_image("p1", "img2", s)
|
||||
assert loc["total"] == 5
|
||||
page = await svc.list_images("p1", "all", s, limit=2, offset=loc["offset"])
|
||||
assert page["items"][0]["id"] == "img2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locate_missing_image_is_404(db):
|
||||
from fastapi import HTTPException
|
||||
await _seed(db, 1)
|
||||
async with db() as s:
|
||||
with pytest.raises(HTTPException) as e:
|
||||
await svc.locate_image("p1", "nope", s)
|
||||
assert e.value.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_limit_is_clamped(db):
|
||||
await _seed(db, 2)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue