feat: add resumable dataset uploads
This commit is contained in:
parent
f8ed08b854
commit
8d40cff227
23 changed files with 913 additions and 21 deletions
|
|
@ -15,6 +15,8 @@ database:
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
bucket_default: danagble
|
bucket_default: danagble
|
||||||
|
# Reference quota for the sidebar meter (R2 free tier = 10 GiB).
|
||||||
|
quota_bytes: 10737418240
|
||||||
# R2 key layout. {project} = project slug, {image} = image id.
|
# R2 key layout. {project} = project slug, {image} = image id.
|
||||||
layout:
|
layout:
|
||||||
image: "datasets/{project}/{image}-{filename}"
|
image: "datasets/{project}/{image}-{filename}"
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,8 @@ database:
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
bucket_default: danagble
|
bucket_default: danagble
|
||||||
|
# Reference quota for the sidebar meter (R2 free tier = 10 GiB).
|
||||||
|
quota_bytes: 10737418240
|
||||||
# R2 key layout. {project} = project slug, {image} = image id.
|
# R2 key layout. {project} = project slug, {image} = image id.
|
||||||
layout:
|
layout:
|
||||||
image: "datasets/{project}/{image}-{filename}"
|
image: "datasets/{project}/{image}-{filename}"
|
||||||
|
|
|
||||||
43
docs/API.md
43
docs/API.md
|
|
@ -24,6 +24,7 @@ Interactive docs: `http://localhost:8000/docs`.
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| GET | `/health` | Liveness incl. DB check, `{ ok: true }` |
|
| GET | `/health` | Liveness incl. DB check, `{ ok: true }` |
|
||||||
| GET | `/metrics` | Phase-1 traffic counters per route |
|
| GET | `/metrics` | Phase-1 traffic counters per route |
|
||||||
|
| GET | `/storage/usage` | Real bucket `{ bytes, objects, quota_bytes }` (10-min cache) |
|
||||||
| GET | `/` (root, no prefix) | Name, docs and health links |
|
| GET | `/` (root, no prefix) | Name, docs and health links |
|
||||||
| POST | `/admin/init-db` | Create tables, seed dev admin. Dev-only |
|
| POST | `/admin/init-db` | Create tables, seed dev admin. Dev-only |
|
||||||
| POST | `/admin/backfill-thumbnails?limit=100` | Mint missing thumbs. Dev-only |
|
| POST | `/admin/backfill-thumbnails?limit=100` | Mint missing thumbs. Dev-only |
|
||||||
|
|
@ -97,6 +98,44 @@ GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0
|
||||||
|
|
||||||
`PATCH /projects/{id}/images/{imageId}?status=completed` — status transition.
|
`PATCH /projects/{id}/images/{imageId}?status=completed` — status transition.
|
||||||
|
|
||||||
|
`POST /projects/{id}/images/batch-presign` — one call for ≤200 files:
|
||||||
|
`[{ fileName, contentType }]` → `[{ fileName, putUrl, r2Key }]`
|
||||||
|
(keys carry unique tmp ids, so duplicate filenames never collide).
|
||||||
|
|
||||||
|
`GET /projects/{id}/images/by-id/{imageId}` — `{ offset, total }` in list
|
||||||
|
order (deterministic `created_at DESC, id DESC`); backs `?imageId=` deep
|
||||||
|
links into thousand-image datasets.
|
||||||
|
|
||||||
|
## Resumable multipart uploads (files ~100MB and up)
|
||||||
|
|
||||||
|
Small files keep the single-PUT presign path. Big single files
|
||||||
|
(GB zips, TIFF, video) go multipart, straight to R2:
|
||||||
|
|
||||||
|
1. `POST /projects/{id}/uploads/multipart` `{ fileName, contentType }` →
|
||||||
|
`{ key, uploadId, partSize: 8388608 }`.
|
||||||
|
2. `POST .../uploads/multipart/parts` `{ key, uploadId, numbers: [1..N] }`
|
||||||
|
→ `{ urls: { "1": "...", ... } }` (1h TTL each).
|
||||||
|
3. PUT every part (parallel, per-part retry; store ETags client-side —
|
||||||
|
the upload resumes after network drops instead of restarting).
|
||||||
|
4. `POST .../uploads/multipart/complete` `{ key, uploadId,
|
||||||
|
parts: [{ ETag, PartNumber }] }` → `{ key, etag }`.
|
||||||
|
5. `POST .../uploads/multipart/abort` to cancel and free the upload.
|
||||||
|
|
||||||
|
ETag response headers are exposed via the bucket CORS policy, which the
|
||||||
|
frontend needs to read them.
|
||||||
|
|
||||||
|
## Server-side dataset imports (GB-scale zips)
|
||||||
|
|
||||||
|
`POST /projects/{id}/imports/local-zip` `{ path }` (absolute server path)
|
||||||
|
or `POST /projects/{id}/imports/r2-zip` `{ r2_key }` (zip already uploaded,
|
||||||
|
e.g. via multipart) → `202 { id, status: queued, ... }`. The worker streams
|
||||||
|
the archive (one image in RAM at a time), uploads originals + 512/128px
|
||||||
|
thumbs concurrently, commits DB rows in batches, then bulk-upserts COCO
|
||||||
|
annotations. Re-running skips files already present.
|
||||||
|
|
||||||
|
`GET /projects/{id}/imports` and `GET /projects/{id}/imports/{jobId}` poll
|
||||||
|
`{ status, total_files, done_files, saved_annotations, error }`.
|
||||||
|
|
||||||
## Annotations (auth)
|
## Annotations (auth)
|
||||||
|
|
||||||
`GET /projects/{pid}/images/{iid}/annotation` — one row per image:
|
`GET /projects/{pid}/images/{iid}/annotation` — one row per image:
|
||||||
|
|
@ -109,6 +148,10 @@ GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0
|
||||||
"visible": true, "bbox": { "x": 10, "y": 20, "width": 100, "height": 80 } }] }
|
"visible": true, "bbox": { "x": 10, "y": 20, "width": 100, "height": 80 } }] }
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`PUT /projects/{id}/annotations/batch` with `{ [imageId]: { classificationLabelId?, flags?, shapes[] } }`
|
||||||
|
upserts up to 500 images in one commit (COCO/YOLO import path) →
|
||||||
|
`{ saved, missing: [...] }`.
|
||||||
|
|
||||||
`PUT` same path with `{ classificationLabelId?, flags?, shapes[] }` upserts
|
`PUT` same path with `{ classificationLabelId?, flags?, shapes[] }` upserts
|
||||||
(one row per image, 404 when the image does not exist) and syncs
|
(one row per image, 404 when the image does not exist) and syncs
|
||||||
`object_count` / flags onto the image row. Also emits an `annotation.save`
|
`object_count` / flags onto the image row. Also emits an `annotation.save`
|
||||||
|
|
|
||||||
|
|
@ -5,10 +5,11 @@ from models.tables import (
|
||||||
Annotation,
|
Annotation,
|
||||||
ExportJob,
|
ExportJob,
|
||||||
Image,
|
Image,
|
||||||
|
ImportJob,
|
||||||
LabelClass,
|
LabelClass,
|
||||||
Project,
|
Project,
|
||||||
User,
|
User,
|
||||||
)
|
)
|
||||||
|
|
||||||
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
|
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
|
||||||
"LabelClass", "Project", "User"]
|
"ImportJob", "LabelClass", "Project", "User"]
|
||||||
|
|
|
||||||
|
|
@ -104,6 +104,22 @@ class ActionEvent(Base):
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
|
||||||
|
class ImportJob(Base):
|
||||||
|
"""Server-side dataset import (e.g. 3GB zip): polled by the UI/admin."""
|
||||||
|
__tablename__ = "imports"
|
||||||
|
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||||
|
project_id: Mapped[str] = mapped_column(String(32), index=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(32), default="zip-coco")
|
||||||
|
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
|
||||||
|
source: Mapped[str] = mapped_column(String(1024), default="")
|
||||||
|
total_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
done_files: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
saved_annotations: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
|
||||||
class ExportJob(Base):
|
class ExportJob(Base):
|
||||||
__tablename__ = "exports"
|
__tablename__ = "exports"
|
||||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
|
||||||
|
|
|
||||||
61
shared/r2.py
61
shared/r2.py
|
|
@ -64,6 +64,67 @@ def get_bytes(key: str) -> bytes:
|
||||||
return obj["Body"].read()
|
return obj["Body"].read()
|
||||||
|
|
||||||
|
|
||||||
|
def bucket_usage(prefix: str = "") -> dict:
|
||||||
|
"""Sum bytes + object count via paginated LIST (no download)."""
|
||||||
|
s = get_settings()
|
||||||
|
total_bytes, total_objects = 0, 0
|
||||||
|
paginator = _client().get_paginator("list_objects_v2")
|
||||||
|
for page in paginator.paginate(Bucket=s.r2_bucket, Prefix=prefix):
|
||||||
|
for obj in page.get("Contents", []):
|
||||||
|
total_bytes += obj.get("Size", 0)
|
||||||
|
total_objects += 1
|
||||||
|
return {"bytes": total_bytes, "objects": total_objects}
|
||||||
|
|
||||||
|
|
||||||
|
def create_multipart(key: str, content_type: str = "application/octet-stream") -> str:
|
||||||
|
"""Start a multipart upload; returns upload_id. Parts upload in parallel,
|
||||||
|
each retriable; the upload survives network drops via the stored id."""
|
||||||
|
s = get_settings()
|
||||||
|
res = _client().create_multipart_upload(
|
||||||
|
Bucket=s.r2_bucket, Key=key, ContentType=content_type)
|
||||||
|
return res["UploadId"]
|
||||||
|
|
||||||
|
|
||||||
|
def presign_part(key: str, upload_id: str, part_number: int,
|
||||||
|
expires_in: int = 3600) -> str:
|
||||||
|
s = get_settings()
|
||||||
|
return _client().generate_presigned_url(
|
||||||
|
"upload_part",
|
||||||
|
Params={"Bucket": s.r2_bucket, "Key": key,
|
||||||
|
"UploadId": upload_id, "PartNumber": part_number},
|
||||||
|
ExpiresIn=expires_in)
|
||||||
|
|
||||||
|
|
||||||
|
def complete_multipart(key: str, upload_id: str, parts: list[dict]) -> str:
|
||||||
|
"""parts: [{ETag, PartNumber}]. Returns final ETag."""
|
||||||
|
s = get_settings()
|
||||||
|
res = _client().complete_multipart_upload(
|
||||||
|
Bucket=s.r2_bucket, Key=key, UploadId=upload_id,
|
||||||
|
MultipartUpload={"Parts": [{"ETag": p["ETag"], "PartNumber": p["PartNumber"]}
|
||||||
|
for p in sorted(parts, key=lambda x: x["PartNumber"])]})
|
||||||
|
return res.get("ETag", "")
|
||||||
|
|
||||||
|
|
||||||
|
def abort_multipart(key: str, upload_id: str) -> None:
|
||||||
|
s = get_settings()
|
||||||
|
try:
|
||||||
|
_client().abort_multipart_upload(Bucket=s.r2_bucket, Key=key, UploadId=upload_id)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def download_to_file(key: str, dest_path: str, chunk_bytes: int = 8 * 1024 * 1024) -> int:
|
||||||
|
"""Stream an object to disk (GB-scale safe, bounded RAM). Returns bytes."""
|
||||||
|
s = get_settings()
|
||||||
|
body = _client().get_object(Bucket=s.r2_bucket, Key=key)["Body"]
|
||||||
|
total = 0
|
||||||
|
with open(dest_path, "wb") as f:
|
||||||
|
for chunk in body.iter_chunks(chunk_size=chunk_bytes):
|
||||||
|
f.write(chunk)
|
||||||
|
total += len(chunk)
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
|
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
|
||||||
cache_control: str | None = None) -> None:
|
cache_control: str | None = None) -> None:
|
||||||
s = get_settings()
|
s = get_settings()
|
||||||
|
|
|
||||||
|
|
@ -73,3 +73,14 @@ class ImageConfirm(BaseModel):
|
||||||
class PresignRequest(BaseModel):
|
class PresignRequest(BaseModel):
|
||||||
fileName: str
|
fileName: str
|
||||||
contentType: str = "image/jpeg"
|
contentType: str = "image/jpeg"
|
||||||
|
|
||||||
|
|
||||||
|
class BatchPresignItem(BaseModel):
|
||||||
|
fileName: str
|
||||||
|
contentType: str = "image/jpeg"
|
||||||
|
|
||||||
|
|
||||||
|
class BulkAnnotationItem(BaseModel):
|
||||||
|
classificationLabelId: str | None = None
|
||||||
|
flags: Flags = Field(default_factory=Flags)
|
||||||
|
shapes: list[AnnotationShape] = Field(default_factory=list)
|
||||||
|
|
|
||||||
|
|
@ -71,3 +71,11 @@ async def invite(body: InviteIn, db: AsyncSession = Depends(get_session)):
|
||||||
async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)):
|
async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)):
|
||||||
from src.services import users as svc
|
from src.services import users as svc
|
||||||
return await svc.patch_user(user_id, body.model_dump(), db)
|
return await svc.patch_user(user_id, body.model_dump(), db)
|
||||||
|
|
||||||
|
|
||||||
|
@users_router.delete("/{user_id}")
|
||||||
|
async def delete_user(user_id: str, db: AsyncSession = Depends(get_session),
|
||||||
|
user = Depends(get_current_user)):
|
||||||
|
from src.services import users as svc
|
||||||
|
actor = user.id if user is not None else None
|
||||||
|
return await svc.delete_user(user_id, actor, db)
|
||||||
|
|
|
||||||
|
|
@ -3,13 +3,21 @@ from __future__ import annotations
|
||||||
from fastapi import APIRouter, Depends, Query
|
from fastapi import APIRouter, Depends, Query
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from shared.schemas import ImageConfirm, PresignRequest
|
from shared.schemas import BatchPresignItem, ImageConfirm, PresignRequest
|
||||||
from src.core.db import get_session
|
from src.core.db import get_session
|
||||||
from src.services import images as svc
|
from src.services import images as svc
|
||||||
|
|
||||||
router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"])
|
router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/by-id/{image_id}")
|
||||||
|
async def locate_image(project_id: str, image_id: str,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
"""Deep-link support (?imageId=): offset of one image in list order."""
|
||||||
|
from src.services import images as svc
|
||||||
|
return await svc.locate_image(project_id, image_id, db)
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
async def list_images(project_id: str, status: str | None = Query(default=None),
|
async def list_images(project_id: str, status: str | None = Query(default=None),
|
||||||
search: str | None = Query(default=None),
|
search: str | None = Query(default=None),
|
||||||
|
|
@ -26,6 +34,13 @@ async def presign_upload(project_id: str, body: PresignRequest):
|
||||||
return svc.build_presign(project_id, body.fileName, body.contentType)
|
return svc.build_presign(project_id, body.fileName, body.contentType)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/batch-presign")
|
||||||
|
async def batch_presign_upload(project_id: str, body: list[BatchPresignItem],
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
"""One call for a whole upload batch (≤200 files)."""
|
||||||
|
return await svc.batch_presign(project_id, [b.model_dump() for b in body], db)
|
||||||
|
|
||||||
|
|
||||||
@router.post("", status_code=201)
|
@router.post("", status_code=201)
|
||||||
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)):
|
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)):
|
||||||
return await svc.confirm_image(project_id, body, db)
|
return await svc.confirm_image(project_id, body, db)
|
||||||
|
|
|
||||||
42
src/api/v1/imports.py
Normal file
42
src/api/v1/imports.py
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from src.core.db import get_session
|
||||||
|
from src.services import imports as svc
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/projects/{project_id}/imports", tags=["imports"])
|
||||||
|
|
||||||
|
|
||||||
|
class LocalZipIn(BaseModel):
|
||||||
|
path: str
|
||||||
|
|
||||||
|
|
||||||
|
class R2ZipIn(BaseModel):
|
||||||
|
r2_key: str
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/local-zip", status_code=202)
|
||||||
|
async def start_local_zip(project_id: str, body: LocalZipIn,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
"""Queue a server-side zip import (GB-scale, streamed). Poll the job id."""
|
||||||
|
return await svc.start_local_zip(project_id, body.path, db)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/r2-zip", status_code=202)
|
||||||
|
async def start_r2_zip(project_id: str, body: R2ZipIn,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
"""Queue import of a zip already uploaded to R2 (e.g. via multipart)."""
|
||||||
|
return await svc.start_r2_zip(project_id, body.r2_key, db)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("")
|
||||||
|
async def list_imports(project_id: str, db: AsyncSession = Depends(get_session)):
|
||||||
|
return await svc.list_jobs(project_id, db)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{job_id}")
|
||||||
|
async def get_import(project_id: str, job_id: str, db: AsyncSession = Depends(get_session)):
|
||||||
|
return await svc.get_job(project_id, job_id, db)
|
||||||
|
|
@ -3,8 +3,9 @@ from __future__ import annotations
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from shared.schemas import LabelClassIn, ProjectCreate
|
from shared.schemas import AnnotationUpsert, BulkAnnotationItem, LabelClassIn, ProjectCreate
|
||||||
from src.core.db import get_session
|
from src.core.db import get_session
|
||||||
|
from src.services import annotations as annotations_svc
|
||||||
from src.services import projects as svc
|
from src.services import projects as svc
|
||||||
|
|
||||||
router = APIRouter(prefix="/projects", tags=["projects"])
|
router = APIRouter(prefix="/projects", tags=["projects"])
|
||||||
|
|
@ -35,6 +36,14 @@ async def delete_project(project_id: str, db: AsyncSession = Depends(get_session
|
||||||
return await svc.delete_project(project_id, db)
|
return await svc.delete_project(project_id, db)
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{project_id}/annotations/batch")
|
||||||
|
async def bulk_upsert_annotations(project_id: str, body: dict[str, BulkAnnotationItem],
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
"""COCO/YOLO import: upsert up to 500 images in one commit."""
|
||||||
|
items = {image_id: AnnotationUpsert(**item.model_dump()) for image_id, item in body.items()}
|
||||||
|
return await annotations_svc.bulk_upsert(project_id, items, db)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{project_id}/classes", status_code=201)
|
@router.post("/{project_id}/classes", status_code=201)
|
||||||
async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)):
|
async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)):
|
||||||
return await svc.add_label_class(project_id, body, db)
|
return await svc.add_label_class(project_id, body, db)
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
from sqlalchemy import select, text
|
from sqlalchemy import select, text
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
|
@ -22,6 +22,28 @@ async def metrics():
|
||||||
return get_snapshot()
|
return get_snapshot()
|
||||||
|
|
||||||
|
|
||||||
|
_usage_cache: dict = {"at": 0.0, "data": None}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/storage/usage")
|
||||||
|
async def storage_usage():
|
||||||
|
"""Real bucket bytes/objects for the sidebar meter (10-min cache)."""
|
||||||
|
import time
|
||||||
|
from shared import r2
|
||||||
|
from src.core.config import get_settings
|
||||||
|
now = time.time()
|
||||||
|
if _usage_cache["data"] is not None and now - _usage_cache["at"] < 600:
|
||||||
|
return _usage_cache["data"]
|
||||||
|
try:
|
||||||
|
usage = r2.bucket_usage()
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(503, f"storage unreachable: {e}")
|
||||||
|
data = {"bytes": usage["bytes"], "objects": usage["objects"],
|
||||||
|
"quota_bytes": get_settings().storage_quota_bytes}
|
||||||
|
_usage_cache.update({"at": now, "data": data})
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
@router.post("/admin/backfill-thumbnails")
|
@router.post("/admin/backfill-thumbnails")
|
||||||
async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)):
|
async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)):
|
||||||
"""Mint thumbs for images uploaded before thumbnailing existed. Dev-only."""
|
"""Mint thumbs for images uploaded before thumbnailing existed. Dev-only."""
|
||||||
|
|
|
||||||
102
src/api/v1/uploads.py
Normal file
102
src/api/v1/uploads.py
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
"""Resumable multipart uploads: big single files (GB zip/TIFF/video).
|
||||||
|
|
||||||
|
Flow: POST /multipart -> {key, uploadId} -> PUT each part to its presigned
|
||||||
|
URL (parallel, retried per part, resumable via stored ETags) ->
|
||||||
|
POST /multipart/complete. Use for anything over ~100MB; small files keep
|
||||||
|
the single-PUT presign path.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from models import Project
|
||||||
|
from shared import r2
|
||||||
|
from src.core.db import get_session
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/projects/{project_id}/uploads", tags=["uploads"])
|
||||||
|
|
||||||
|
PART_SIZE = 8 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
class MultipartStart(BaseModel):
|
||||||
|
fileName: str
|
||||||
|
contentType: str = "application/octet-stream"
|
||||||
|
|
||||||
|
|
||||||
|
class MultipartParts(BaseModel):
|
||||||
|
key: str
|
||||||
|
uploadId: str
|
||||||
|
numbers: list[int] = Field(max_length=1000)
|
||||||
|
|
||||||
|
|
||||||
|
class PartETag(BaseModel):
|
||||||
|
ETag: str
|
||||||
|
PartNumber: int
|
||||||
|
|
||||||
|
|
||||||
|
class MultipartComplete(BaseModel):
|
||||||
|
key: str
|
||||||
|
uploadId: str
|
||||||
|
parts: list[PartETag]
|
||||||
|
|
||||||
|
|
||||||
|
class MultipartAbort(BaseModel):
|
||||||
|
key: str
|
||||||
|
uploadId: str
|
||||||
|
|
||||||
|
|
||||||
|
async def _project_slug(project_id: str, db: AsyncSession) -> str:
|
||||||
|
p = await db.get(Project, project_id)
|
||||||
|
if p is None:
|
||||||
|
raise HTTPException(404, "project not found")
|
||||||
|
return p.slug
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/multipart")
|
||||||
|
async def start_multipart(project_id: str, body: MultipartStart,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
import uuid as _uuid
|
||||||
|
slug = await _project_slug(project_id, db)
|
||||||
|
key = f"uploads/{slug}/{_uuid.uuid4().hex[:12]}-{body.fileName.replace('/', '_')}"
|
||||||
|
try:
|
||||||
|
upload_id = r2.create_multipart(key, body.contentType)
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(503, f"storage unreachable: {e}")
|
||||||
|
return {"key": key, "uploadId": upload_id, "partSize": PART_SIZE}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/multipart/parts")
|
||||||
|
async def part_urls(project_id: str, body: MultipartParts,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
await _project_slug(project_id, db)
|
||||||
|
if any(n < 1 or n > 10000 for n in body.numbers):
|
||||||
|
raise HTTPException(400, "invalid part numbers")
|
||||||
|
try:
|
||||||
|
urls = {n: r2.presign_part(body.key, body.uploadId, n) for n in body.numbers}
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(503, f"storage unreachable: {e}")
|
||||||
|
return {"urls": urls}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/multipart/complete")
|
||||||
|
async def complete_multipart(project_id: str, body: MultipartComplete,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
await _project_slug(project_id, db)
|
||||||
|
try:
|
||||||
|
etag = r2.complete_multipart(body.key, body.uploadId,
|
||||||
|
[p.model_dump() for p in body.parts])
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(400, f"complete failed (missing/extra parts?): {e}")
|
||||||
|
if not r2.head_exists(body.key):
|
||||||
|
raise HTTPException(400, "object not found after complete")
|
||||||
|
return {"key": body.key, "etag": etag}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/multipart/abort")
|
||||||
|
async def abort_multipart(project_id: str, body: MultipartAbort,
|
||||||
|
db: AsyncSession = Depends(get_session)):
|
||||||
|
await _project_slug(project_id, db)
|
||||||
|
r2.abort_multipart(body.key, body.uploadId)
|
||||||
|
return {"aborted": True}
|
||||||
|
|
@ -46,6 +46,7 @@ class Settings:
|
||||||
layout = storage.get("layout", {})
|
layout = storage.get("layout", {})
|
||||||
presigned = storage.get("presigned_ttl", {})
|
presigned = storage.get("presigned_ttl", {})
|
||||||
self.r2_layout: dict = layout
|
self.r2_layout: dict = layout
|
||||||
|
self.storage_quota_bytes: int = int(storage.get("quota_bytes", 10 * 1024**3))
|
||||||
thumb = storage.get("thumbnail", {})
|
thumb = storage.get("thumbnail", {})
|
||||||
self.thumbnail_enabled: bool = bool(thumb.get("enabled", True))
|
self.thumbnail_enabled: bool = bool(thumb.get("enabled", True))
|
||||||
self.thumbnail_max_side: int = int(thumb.get("max_side", 512))
|
self.thumbnail_max_side: int = int(thumb.get("max_side", 512))
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from shared import events
|
||||||
from shared.logger import get_logger
|
from shared.logger import get_logger
|
||||||
from shared.tracing import TraceMiddleware
|
from shared.tracing import TraceMiddleware
|
||||||
from shared.traffic import TrafficMiddleware
|
from shared.traffic import TrafficMiddleware
|
||||||
from src.api.v1 import activities, annotations, auth, events as events_api, images, projects, system
|
from src.api.v1 import activities, annotations, auth, events as events_api, images, imports, projects, system, uploads
|
||||||
from src.api.v1.auth import users_router
|
from src.api.v1.auth import users_router
|
||||||
from src.core.config import get_settings
|
from src.core.config import get_settings
|
||||||
from src.core.security import get_current_user
|
from src.core.security import get_current_user
|
||||||
|
|
@ -61,6 +61,8 @@ app.include_router(activities.router, prefix=prefix, dependencies=authed)
|
||||||
app.include_router(projects.router, prefix=prefix, dependencies=authed)
|
app.include_router(projects.router, prefix=prefix, dependencies=authed)
|
||||||
app.include_router(images.router, prefix=prefix, dependencies=authed)
|
app.include_router(images.router, prefix=prefix, dependencies=authed)
|
||||||
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
|
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
|
||||||
|
app.include_router(imports.router, prefix=prefix, dependencies=authed)
|
||||||
|
app.include_router(uploads.router, prefix=prefix, dependencies=authed)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import time
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from models import Annotation, Image
|
from models import Annotation, Image, Project
|
||||||
from shared import events
|
from shared import events
|
||||||
from shared.schemas import AnnotationUpsert
|
from shared.schemas import AnnotationUpsert
|
||||||
|
|
||||||
|
|
@ -29,10 +29,8 @@ async def get_annotation(project_id: str, image_id: str, db: AsyncSession) -> di
|
||||||
"flags": ann.flags, "shapes": ann.shapes}
|
"flags": ann.flags, "shapes": ann.shapes}
|
||||||
|
|
||||||
|
|
||||||
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
|
async def _apply_upsert(db: AsyncSession, project_id: str, image_id: str,
|
||||||
db: AsyncSession) -> dict:
|
body: AnnotationUpsert) -> int:
|
||||||
im = await _require_image(project_id, image_id, db)
|
|
||||||
t0 = time.perf_counter()
|
|
||||||
ann = await db.get(Annotation, image_id)
|
ann = await db.get(Annotation, image_id)
|
||||||
payload = {"project_id": project_id,
|
payload = {"project_id": project_id,
|
||||||
"classification_label_id": body.classificationLabelId,
|
"classification_label_id": body.classificationLabelId,
|
||||||
|
|
@ -49,6 +47,43 @@ async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpse
|
||||||
im.object_count = len(body.shapes)
|
im.object_count = len(body.shapes)
|
||||||
im.classification_label_id = body.classificationLabelId
|
im.classification_label_id = body.classificationLabelId
|
||||||
im.flags = body.flags.model_dump()
|
im.flags = body.flags.model_dump()
|
||||||
|
return len(body.shapes)
|
||||||
|
|
||||||
|
|
||||||
|
async def bulk_upsert(project_id: str, items: dict[str, AnnotationUpsert],
|
||||||
|
db: AsyncSession) -> dict:
|
||||||
|
"""COCO/YOLO import path: one commit for the whole batch (≤500 images).
|
||||||
|
|
||||||
|
Imported labels are machine data awaiting human review, so saved rows go
|
||||||
|
to pending_review — that is what the review queue and stats count on.
|
||||||
|
"""
|
||||||
|
from fastapi import HTTPException
|
||||||
|
if await db.get(Project, project_id) is None:
|
||||||
|
raise HTTPException(404, "project not found")
|
||||||
|
from shared import events
|
||||||
|
import time
|
||||||
|
saved, missing = 0, []
|
||||||
|
t0 = time.perf_counter()
|
||||||
|
for image_id, body in list(items.items())[:500]:
|
||||||
|
im = await db.get(Image, image_id)
|
||||||
|
if im is None or im.project_id != project_id:
|
||||||
|
missing.append(image_id)
|
||||||
|
continue
|
||||||
|
await _apply_upsert(db, project_id, image_id, body)
|
||||||
|
im.status = "pending_review"
|
||||||
|
saved += 1
|
||||||
|
await db.commit()
|
||||||
|
await events.emit("dataset.import", project_id=project_id,
|
||||||
|
payload={"saved": saved, "missing": len(missing)},
|
||||||
|
latency_ms=(time.perf_counter() - t0) * 1000)
|
||||||
|
return {"saved": saved, "missing": missing}
|
||||||
|
|
||||||
|
|
||||||
|
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
|
||||||
|
db: AsyncSession) -> dict:
|
||||||
|
await _require_image(project_id, image_id, db)
|
||||||
|
t0 = time.perf_counter()
|
||||||
|
await _apply_upsert(db, project_id, image_id, body)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
await events.emit("annotation.save", project_id=project_id, image_id=image_id,
|
await events.emit("annotation.save", project_id=project_id, image_id=image_id,
|
||||||
payload={"shapeCount": len(body.shapes),
|
payload={"shapeCount": len(body.shapes),
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from fastapi import HTTPException
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from models import Image
|
from models import Image, Project
|
||||||
from shared import events, keys, r2
|
from shared import events, keys, r2
|
||||||
from shared.schemas import ImageConfirm
|
from shared.schemas import ImageConfirm
|
||||||
|
|
||||||
|
|
@ -40,7 +40,10 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession,
|
||||||
q = q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
|
q = q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
|
||||||
count_q = count_q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
|
count_q = count_q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
|
||||||
total = (await db.execute(count_q)).scalar() or 0
|
total = (await db.execute(count_q)).scalar() or 0
|
||||||
rows = (await db.execute(q.order_by(Image.created_at.desc()).limit(limit).offset(offset))).scalars().all()
|
# Deterministic order (id tiebreak): pages must not shift underfoot.
|
||||||
|
rows = (await db.execute(
|
||||||
|
q.order_by(Image.created_at.desc(), Image.id.desc()).limit(limit).offset(offset)
|
||||||
|
)).scalars().all()
|
||||||
users = (await db.execute(select(User))).scalars().all()
|
users = (await db.execute(select(User))).scalars().all()
|
||||||
names = {u.id: u.name for u in users}
|
names = {u.id: u.name for u in users}
|
||||||
items = [{"id": im.id, "projectId": im.project_id, "fileName": im.file_name,
|
items = [{"id": im.id, "projectId": im.project_id, "fileName": im.file_name,
|
||||||
|
|
@ -57,12 +60,44 @@ async def list_images(project_id: str, status: str | None, db: AsyncSession,
|
||||||
return {"items": items, "total": total}
|
return {"items": items, "total": total}
|
||||||
|
|
||||||
|
|
||||||
|
async def locate_image(project_id: str, image_id: str, db: AsyncSession) -> dict:
|
||||||
|
"""Offset (0-based) of one image in list order, for ?imageId= deep links."""
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from sqlalchemy import func, text
|
||||||
|
total = (await db.execute(
|
||||||
|
select(func.count()).where(Image.project_id == project_id))).scalar() or 0
|
||||||
|
row = (await db.execute(text(
|
||||||
|
"SELECT rn FROM (SELECT id, ROW_NUMBER() OVER "
|
||||||
|
"(ORDER BY created_at DESC, id DESC) AS rn FROM images "
|
||||||
|
"WHERE project_id = :pid) t WHERE id = :iid"),
|
||||||
|
{"pid": project_id, "iid": image_id})).first()
|
||||||
|
if row is None:
|
||||||
|
raise HTTPException(404, "image not found")
|
||||||
|
return {"offset": int(row[0]) - 1, "total": total}
|
||||||
|
|
||||||
|
|
||||||
def build_presign(project_id: str, file_name: str, content_type: str) -> dict:
|
def build_presign(project_id: str, file_name: str, content_type: str) -> dict:
|
||||||
"""Step 1/2 direct upload: frontend PUTs bytes straight to R2 with this URL."""
|
"""Step 1/2 direct upload: frontend PUTs bytes straight to R2 with this URL."""
|
||||||
key = keys.image_key(project_id, "tmp", file_name) # real id assigned on confirm
|
import uuid as _uuid
|
||||||
|
# Unique tmp id per file: duplicate filenames in one batch must not collide.
|
||||||
|
key = keys.image_key(project_id, f"tmp-{_uuid.uuid4().hex[:8]}", file_name)
|
||||||
return {"putUrl": r2.presign_put(key, content_type), "r2Key": key}
|
return {"putUrl": r2.presign_put(key, content_type), "r2Key": key}
|
||||||
|
|
||||||
|
|
||||||
|
async def batch_presign(project_id: str, files: list[dict], db: AsyncSession) -> list[dict]:
|
||||||
|
"""One call for a whole batch (≤200): avoids a presign roundtrip per file."""
|
||||||
|
if await db.get(Project, project_id) is None:
|
||||||
|
from fastapi import HTTPException
|
||||||
|
raise HTTPException(404, "project not found")
|
||||||
|
out = []
|
||||||
|
for f in files[:200]:
|
||||||
|
slot = build_presign(project_id, f.get("fileName", "upload.jpg"),
|
||||||
|
f.get("contentType", "image/jpeg"))
|
||||||
|
slot["fileName"] = f.get("fileName", "upload.jpg")
|
||||||
|
out.append(slot)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession) -> dict:
|
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession) -> dict:
|
||||||
"""Step 2/2: metadata row after the R2 PUT succeeds."""
|
"""Step 2/2: metadata row after the R2 PUT succeeds."""
|
||||||
if not r2.head_exists(body.r2Key):
|
if not r2.head_exists(body.r2Key):
|
||||||
|
|
|
||||||
289
src/services/imports.py
Normal file
289
src/services/imports.py
Normal file
|
|
@ -0,0 +1,289 @@
|
||||||
|
"""Server-side dataset import: stream a big zip (GBs) without loading it to RAM.
|
||||||
|
|
||||||
|
One image in memory at a time for reads; network puts run concurrently;
|
||||||
|
DB commits in batches of 100. Re-runnable: files already in the project
|
||||||
|
(by file_name) are skipped.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from models import Image, ImportJob, LabelClass, Project
|
||||||
|
from shared import events, keys, r2
|
||||||
|
from shared.logger import get_logger
|
||||||
|
from shared.schemas import AnnotationShape, AnnotationUpsert, Flags
|
||||||
|
from shared.thumbnails import IMMUTABLE, resize_to_thumbnail
|
||||||
|
from src.core.db import get_session_factory
|
||||||
|
|
||||||
|
log = get_logger("danable.imports")
|
||||||
|
|
||||||
|
IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp", ".bmp")
|
||||||
|
PUT_CONCURRENCY = 24
|
||||||
|
DB_BATCH = 100
|
||||||
|
|
||||||
|
|
||||||
|
def _basename(name: str) -> str:
|
||||||
|
return name.split("/")[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def _is_image(name: str) -> bool:
|
||||||
|
return name.lower().endswith(IMAGE_EXTS)
|
||||||
|
|
||||||
|
|
||||||
|
def build_bulk_from_coco(coco: dict, file_to_image: dict[str, str],
|
||||||
|
cat_to_class: dict[str, dict]) -> dict[str, AnnotationUpsert]:
|
||||||
|
"""Pure mapping: COCO annotations -> bulk-upsert payload (unit-tested)."""
|
||||||
|
cat_name = {str(c.get("id")): c.get("name", f"class_{c.get('id')}")
|
||||||
|
for c in coco.get("categories", [])}
|
||||||
|
by_image: dict[str, list] = {}
|
||||||
|
for ann in coco.get("annotations", []):
|
||||||
|
meta = next((i for i in coco.get("images", [])
|
||||||
|
if str(i.get("id")) == str(ann.get("image_id"))), None)
|
||||||
|
if meta is None:
|
||||||
|
continue
|
||||||
|
fname = _basename(meta.get("file_name", ""))
|
||||||
|
image_id = file_to_image.get(fname)
|
||||||
|
if image_id is None:
|
||||||
|
continue
|
||||||
|
label = cat_name.get(str(ann.get("category_id")), "unknown")
|
||||||
|
cls = cat_to_class.get(label.lower())
|
||||||
|
if cls is None:
|
||||||
|
continue
|
||||||
|
shape = {"labelId": cls["id"], "labelName": cls["name"], "color": cls["color"]}
|
||||||
|
seg = ann.get("segmentation")
|
||||||
|
if isinstance(seg, list) and seg and isinstance(seg[0], list) and len(seg[0]) >= 6:
|
||||||
|
ring = seg[0]
|
||||||
|
pts = [{"x": int(ring[i]), "y": int(ring[i + 1])}
|
||||||
|
for i in range(0, len(ring) - 1, 2)]
|
||||||
|
shape.update({"type": "polygon", "points": pts})
|
||||||
|
elif isinstance(ann.get("bbox"), list) and len(ann["bbox"]) == 4:
|
||||||
|
x, y, w, h = (int(v) for v in ann["bbox"])
|
||||||
|
shape.update({"type": "bbox", "bbox": {"x": x, "y": y, "width": w, "height": h}})
|
||||||
|
else:
|
||||||
|
continue # RLE/crowd without bbox: skipped
|
||||||
|
by_image.setdefault(image_id, []).append(shape)
|
||||||
|
payload: dict[str, AnnotationUpsert] = {}
|
||||||
|
for image_id, shapes in by_image.items():
|
||||||
|
payload[image_id] = AnnotationUpsert(
|
||||||
|
flags=Flags(),
|
||||||
|
shapes=[AnnotationShape(id=f"zip-{k}", index=k + 1, visible=True, **s)
|
||||||
|
for k, s in enumerate(shapes)],
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _job_to_dict(job: ImportJob) -> dict:
|
||||||
|
return {"id": job.id, "project_id": job.project_id, "kind": job.kind,
|
||||||
|
"status": job.status, "source": job.source,
|
||||||
|
"total_files": job.total_files, "done_files": job.done_files,
|
||||||
|
"saved_annotations": job.saved_annotations, "error": job.error}
|
||||||
|
|
||||||
|
|
||||||
|
async def list_jobs(project_id: str, db: AsyncSession) -> list[dict]:
|
||||||
|
rows = (await db.execute(select(ImportJob).where(
|
||||||
|
ImportJob.project_id == project_id).order_by(ImportJob.created_at.desc()))
|
||||||
|
).scalars().all()
|
||||||
|
return [_job_to_dict(j) for j in rows]
|
||||||
|
|
||||||
|
|
||||||
|
async def get_job(project_id: str, job_id: str, db: AsyncSession) -> dict:
|
||||||
|
job = await db.get(ImportJob, job_id)
|
||||||
|
if job is None or job.project_id != project_id:
|
||||||
|
raise HTTPException(404, "import job not found")
|
||||||
|
return _job_to_dict(job)
|
||||||
|
|
||||||
|
|
||||||
|
async def start_local_zip(project_id: str, path: str, db: AsyncSession) -> dict:
|
||||||
|
p = await db.get(Project, project_id)
|
||||||
|
if p is None:
|
||||||
|
raise HTTPException(404, "project not found")
|
||||||
|
if not os.path.isabs(path) or not path.endswith(".zip") or not os.path.exists(path):
|
||||||
|
raise HTTPException(400, "path must be an existing absolute .zip file")
|
||||||
|
job = ImportJob(project_id=project_id, kind="zip-coco", status="queued", source=path)
|
||||||
|
db.add(job)
|
||||||
|
await db.commit()
|
||||||
|
await db.refresh(job)
|
||||||
|
asyncio.create_task(_run_zip_coco(job.id, project_id, path))
|
||||||
|
return _job_to_dict(job)
|
||||||
|
|
||||||
|
|
||||||
|
async def start_r2_zip(project_id: str, r2_key: str, db: AsyncSession) -> dict:
|
||||||
|
"""Import a zip already in R2 (e.g. multipart-uploaded from the browser):
|
||||||
|
stream it to local temp (bounded RAM), run the standard pipeline, delete."""
|
||||||
|
p = await db.get(Project, project_id)
|
||||||
|
if p is None:
|
||||||
|
raise HTTPException(404, "project not found")
|
||||||
|
if not r2_key.endswith(".zip"):
|
||||||
|
raise HTTPException(400, "r2_key must point to a .zip object")
|
||||||
|
job = ImportJob(project_id=project_id, kind="zip-coco",
|
||||||
|
status="queued", source=f"r2:{r2_key}")
|
||||||
|
db.add(job)
|
||||||
|
await db.commit()
|
||||||
|
await db.refresh(job)
|
||||||
|
asyncio.create_task(_run_r2_zip(job.id, project_id, r2_key))
|
||||||
|
return _job_to_dict(job)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_r2_zip(job_id: str, project_id: str, r2_key: str) -> None:
|
||||||
|
import tempfile
|
||||||
|
tmp = tempfile.mktemp(prefix=f"danable-import-{job_id}-", suffix=".zip")
|
||||||
|
try:
|
||||||
|
await asyncio.to_thread(r2.download_to_file, r2_key, tmp)
|
||||||
|
await _run_zip_coco(job_id, project_id, tmp)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
os.unlink(tmp)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_zip_coco(job_id: str, project_id: str, path: str) -> None:
|
||||||
|
t_all = time.perf_counter()
|
||||||
|
factory = get_session_factory()
|
||||||
|
async with factory() as db:
|
||||||
|
job = await db.get(ImportJob, job_id)
|
||||||
|
if job is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
job.status = "running"
|
||||||
|
await db.commit()
|
||||||
|
project = await db.get(Project, project_id)
|
||||||
|
if project is None:
|
||||||
|
raise RuntimeError("project gone")
|
||||||
|
|
||||||
|
zf = zipfile.ZipFile(path)
|
||||||
|
names = zf.namelist()
|
||||||
|
coco_name = next((n for n in names if _basename(n) == "annotations.json"),
|
||||||
|
next((n for n in names if n.endswith(".json")), None))
|
||||||
|
if coco_name is None:
|
||||||
|
raise RuntimeError("no COCO json found in zip")
|
||||||
|
coco = json.loads(zf.read(coco_name))
|
||||||
|
|
||||||
|
# 1. ensure label classes exist
|
||||||
|
palette = ["#3B82F6", "#10B981", "#F59E0B", "#A855F7", "#EF4444", "#06B6D4"]
|
||||||
|
existing = (await db.execute(select(LabelClass).where(
|
||||||
|
LabelClass.project_id == project_id))).scalars().all()
|
||||||
|
have = {c.name.lower() for c in existing}
|
||||||
|
for i, c in enumerate(coco.get("categories", [])):
|
||||||
|
if c.get("name", "").lower() not in have:
|
||||||
|
db.add(LabelClass(project_id=project_id, name=c["name"],
|
||||||
|
color=palette[(len(existing) + i) % len(palette)]))
|
||||||
|
have.add(c.get("name", "").lower())
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
# 2. stream images: sequential reads, concurrent puts, batched commits
|
||||||
|
present = set((await db.execute(select(Image.file_name).where(
|
||||||
|
Image.project_id == project_id))).scalars().all())
|
||||||
|
entries = [n for n in names if _is_image(n) and _basename(n) not in present]
|
||||||
|
job.total_files = len(entries)
|
||||||
|
await db.commit()
|
||||||
|
log.info("import %s: %d new files of %d", job_id, len(entries), len(names))
|
||||||
|
|
||||||
|
dims = {_basename(i.get("file_name", "")):
|
||||||
|
(i.get("width", 0), i.get("height", 0))
|
||||||
|
for i in coco.get("images", [])}
|
||||||
|
sem = asyncio.Semaphore(PUT_CONCURRENCY)
|
||||||
|
|
||||||
|
async def process_batch(batch: list[tuple[str, bytes]]) -> None:
|
||||||
|
# Concurrent network puts (bounded); then one DB commit.
|
||||||
|
results = await asyncio.gather(*[
|
||||||
|
put_all(
|
||||||
|
keys.image_key(project.slug, f"zip-{uuid.uuid4().hex[:8]}", fname),
|
||||||
|
keys.thumbnail_key(project.slug, fname.rsplit(".", 1)[0]),
|
||||||
|
keys.thumbnail_small_key(project.slug, fname.rsplit(".", 1)[0]),
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
for fname, data in batch
|
||||||
|
])
|
||||||
|
rows = []
|
||||||
|
for (fname, data), keys3 in zip(batch, results):
|
||||||
|
r2_key, thumb_key, small_key = keys3
|
||||||
|
w, h = dims.get(fname, (0, 0))
|
||||||
|
rows.append(Image(
|
||||||
|
project_id=project_id, file_name=fname, r2_key=r2_key,
|
||||||
|
thumbnail_r2_key=thumb_key, thumbnail_small_r2_key=small_key,
|
||||||
|
width=w, height=h, status="unassigned", flags={}))
|
||||||
|
db.add_all(rows)
|
||||||
|
await db.commit()
|
||||||
|
job.done_files = (job.done_files or 0) + len(rows)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
async def put_all(r2_key: str, thumb_key: str, small_key: str,
|
||||||
|
data: bytes) -> tuple[str, str, str]:
|
||||||
|
async with sem:
|
||||||
|
await asyncio.to_thread(r2.put_bytes, r2_key, data, "image/jpeg")
|
||||||
|
try:
|
||||||
|
big = await asyncio.to_thread(resize_to_thumbnail, data, 512, 70)
|
||||||
|
tiny = await asyncio.to_thread(resize_to_thumbnail, data, 128, 65)
|
||||||
|
await asyncio.to_thread(
|
||||||
|
r2.put_bytes, thumb_key, big, "image/jpeg", IMMUTABLE)
|
||||||
|
await asyncio.to_thread(
|
||||||
|
r2.put_bytes, small_key, tiny, "image/jpeg", IMMUTABLE)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("thumb inline failed: %s", e)
|
||||||
|
return (r2_key, thumb_key, small_key)
|
||||||
|
|
||||||
|
batch: list[tuple[str, str]] = []
|
||||||
|
|
||||||
|
async def flush_names() -> None:
|
||||||
|
if not batch:
|
||||||
|
return
|
||||||
|
# Sequential disk reads (zipfile is not thread-safe), then
|
||||||
|
# concurrent network puts for the whole batch at once.
|
||||||
|
loaded = [(fname, zf.read(name)) for name, fname in batch]
|
||||||
|
await process_batch(loaded)
|
||||||
|
batch.clear()
|
||||||
|
|
||||||
|
for name in entries:
|
||||||
|
batch.append((name, _basename(name)))
|
||||||
|
if len(batch) >= 48:
|
||||||
|
await flush_names()
|
||||||
|
await flush_names()
|
||||||
|
|
||||||
|
# 3. annotations via the bulk path in 500-image chunks
|
||||||
|
rows = (await db.execute(select(Image.id, Image.file_name).where(
|
||||||
|
Image.project_id == project_id))).all()
|
||||||
|
file_to_image = {fname: iid for iid, fname in rows}
|
||||||
|
classes = (await db.execute(select(LabelClass).where(
|
||||||
|
LabelClass.project_id == project_id))).scalars().all()
|
||||||
|
cat_to_class = {c.name.lower(): {"id": c.id, "name": c.name, "color": c.color}
|
||||||
|
for c in classes}
|
||||||
|
from src.services import annotations as ann_svc
|
||||||
|
payload = build_bulk_from_coco(coco, file_to_image, cat_to_class)
|
||||||
|
saved = 0
|
||||||
|
chunk: dict[str, AnnotationUpsert] = {}
|
||||||
|
for iid, body in payload.items():
|
||||||
|
chunk[iid] = body
|
||||||
|
if len(chunk) >= 500:
|
||||||
|
out = await ann_svc.bulk_upsert(project_id, chunk, db)
|
||||||
|
saved += out["saved"]
|
||||||
|
chunk = {}
|
||||||
|
if chunk:
|
||||||
|
out = await ann_svc.bulk_upsert(project_id, chunk, db)
|
||||||
|
saved += out["saved"]
|
||||||
|
job.saved_annotations = saved
|
||||||
|
job.status = "done"
|
||||||
|
await db.commit()
|
||||||
|
await events.emit("dataset.import", project_id=project_id,
|
||||||
|
payload={"via": "zip", "files": job.done_files, "saved": saved},
|
||||||
|
latency_ms=(time.perf_counter() - t_all) * 1000)
|
||||||
|
log.info("import %s done: %d files, %d annotations", job_id, job.done_files, saved)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("import %s failed: %s", job_id, e)
|
||||||
|
try:
|
||||||
|
job = await db.get(ImportJob, job_id)
|
||||||
|
if job is not None:
|
||||||
|
job.status = "error"
|
||||||
|
job.error = str(e)[:2000]
|
||||||
|
await db.commit()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
@ -44,16 +44,23 @@ async def create_project(body: ProjectCreate, db: AsyncSession) -> dict:
|
||||||
|
|
||||||
|
|
||||||
async def _stats(project_id: str, db: AsyncSession) -> dict:
|
async def _stats(project_id: str, db: AsyncSession) -> dict:
|
||||||
counts: dict[str, int] = {}
|
"""One GROUP BY instead of five counts + a sum: the list endpoint calls
|
||||||
for st in ("unassigned", "in_progress", "pending_review", "completed"):
|
this per project, so roundtrips dominate at 10+ projects."""
|
||||||
counts[st] = (await db.execute(
|
rows = (await db.execute(
|
||||||
select(func.count()).where(Image.project_id == project_id, Image.status == st))).scalar() or 0
|
select(Image.status, func.count(),
|
||||||
|
func.coalesce(func.sum(Image.object_count), 0))
|
||||||
|
.where(Image.project_id == project_id)
|
||||||
|
.group_by(Image.status))).all()
|
||||||
|
counts = {st: 0 for st in ("unassigned", "in_progress", "pending_review", "completed")}
|
||||||
|
objects = 0
|
||||||
|
for st, n, o in rows:
|
||||||
|
if st in counts:
|
||||||
|
counts[st] = n or 0
|
||||||
|
objects += int(o or 0)
|
||||||
total = sum(counts.values())
|
total = sum(counts.values())
|
||||||
objects = (await db.execute(
|
|
||||||
select(func.coalesce(func.sum(Image.object_count), 0)).where(Image.project_id == project_id))).scalar() or 0
|
|
||||||
return {"totalImages": total, "unassigned": counts["unassigned"], "inProgress": counts["in_progress"],
|
return {"totalImages": total, "unassigned": counts["unassigned"], "inProgress": counts["in_progress"],
|
||||||
"pendingReview": counts["pending_review"], "completed": counts["completed"],
|
"pendingReview": counts["pending_review"], "completed": counts["completed"],
|
||||||
"totalObjects": int(objects)}
|
"totalObjects": objects}
|
||||||
|
|
||||||
|
|
||||||
async def _members(project_id: str, db: AsyncSession) -> list[dict]:
|
async def _members(project_id: str, db: AsyncSession) -> list[dict]:
|
||||||
|
|
@ -94,13 +101,28 @@ async def _thumbnail_url(project_id: str, db: AsyncSession) -> str:
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
async def _class_counts(project_id: str, db: AsyncSession) -> dict[str, int]:
|
||||||
|
"""Per-label object totals straight from stored shapes (always exact,
|
||||||
|
including bulk imports and deletions — no counter drift possible)."""
|
||||||
|
from sqlalchemy import text
|
||||||
|
rows = (await db.execute(text(
|
||||||
|
"SELECT elem->>'labelId' AS lid, COUNT(*) FROM annotations, "
|
||||||
|
"LATERAL json_array_elements(CASE WHEN shapes IS NULL THEN '[]' "
|
||||||
|
"WHEN json_typeof(shapes) = 'array' THEN shapes ELSE '[]' END) AS elem "
|
||||||
|
"WHERE project_id = :pid GROUP BY lid"),
|
||||||
|
{"pid": project_id})).all()
|
||||||
|
return {lid: n for lid, n in rows if lid}
|
||||||
|
|
||||||
|
|
||||||
async def _project_card(p: Project, db: AsyncSession) -> dict:
|
async def _project_card(p: Project, db: AsyncSession) -> dict:
|
||||||
classes = (await db.execute(select(LabelClass).where(LabelClass.project_id == p.id))).scalars().all()
|
classes = (await db.execute(select(LabelClass).where(LabelClass.project_id == p.id))).scalars().all()
|
||||||
|
counts = await _class_counts(p.id, db)
|
||||||
return {"id": p.id, "slug": p.slug, "name": p.name, "taskType": p.task_type,
|
return {"id": p.id, "slug": p.slug, "name": p.name, "taskType": p.task_type,
|
||||||
"status": p.status, "description": p.description,
|
"status": p.status, "description": p.description,
|
||||||
"thumbnailUrl": await _thumbnail_url(p.id, db),
|
"thumbnailUrl": await _thumbnail_url(p.id, db),
|
||||||
"updatedAt": p.updated_at.isoformat() if p.updated_at else "",
|
"updatedAt": p.updated_at.isoformat() if p.updated_at else "",
|
||||||
"classes": [{"id": c.id, "name": c.name, "color": c.color} for c in classes],
|
"classes": [{"id": c.id, "name": c.name, "color": c.color,
|
||||||
|
"count": counts.get(c.id, 0)} for c in classes],
|
||||||
"stats": await _stats(p.id, db),
|
"stats": await _stats(p.id, db),
|
||||||
"members": await _members(p.id, db)}
|
"members": await _members(p.id, db)}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,17 @@ async def invite_user(email: str, role: str, project_ids: list[str], db: AsyncSe
|
||||||
return to_public(u)
|
return to_public(u)
|
||||||
|
|
||||||
|
|
||||||
|
async def delete_user(user_id: str, actor_id: str | None, db: AsyncSession) -> dict:
|
||||||
|
u = await db.get(User, user_id)
|
||||||
|
if u is None:
|
||||||
|
raise HTTPException(404, "user not found")
|
||||||
|
if actor_id is not None and u.id == actor_id:
|
||||||
|
raise HTTPException(400, "cannot delete your own account")
|
||||||
|
await db.delete(u)
|
||||||
|
await db.commit()
|
||||||
|
return {"deleted": user_id}
|
||||||
|
|
||||||
|
|
||||||
async def patch_user(user_id: str, patch: dict, db: AsyncSession) -> dict:
|
async def patch_user(user_id: str, patch: dict, db: AsyncSession) -> dict:
|
||||||
u = await db.get(User, user_id)
|
u = await db.get(User, user_id)
|
||||||
if u is None:
|
if u is None:
|
||||||
|
|
|
||||||
84
tests/test_bulk.py
Normal file
84
tests/test_bulk.py
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
"""Batch upload prep + bulk annotation import (COCO/YOLO path)."""
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
|
from shared.schemas import AnnotationUpsert
|
||||||
|
from src.services import annotations as ann_svc
|
||||||
|
from src.services import images as img_svc
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture
|
||||||
|
async def db():
|
||||||
|
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||||
|
from src.core.db import Base
|
||||||
|
async with engine.begin() as conn:
|
||||||
|
await conn.run_sync(Base.metadata.create_all)
|
||||||
|
yield async_sessionmaker(engine, expire_on_commit=False)
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed(db):
|
||||||
|
async with db() as s:
|
||||||
|
await s.execute(text(
|
||||||
|
"INSERT INTO projects (id, slug, name, task_type, status) "
|
||||||
|
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
|
||||||
|
await s.execute(text(
|
||||||
|
"INSERT INTO images (id, project_id, file_name, r2_key, status, flags) "
|
||||||
|
"VALUES ('img1', 'p1', 'a.jpg', 'k1', 'in_progress', '{}'),"
|
||||||
|
"('img2', 'p1', 'b.jpg', 'k2', 'in_progress', '{}')"))
|
||||||
|
await s.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _body(n=1):
|
||||||
|
from shared.schemas import AnnotationShape
|
||||||
|
return AnnotationUpsert(shapes=[
|
||||||
|
AnnotationShape(id=f"s{i}", index=i + 1, type="bbox", labelId="car",
|
||||||
|
bbox={"x": 1, "y": 2, "width": 10, "height": 10})
|
||||||
|
for i in range(n)])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bulk_upsert_saves_and_reports_missing(db):
|
||||||
|
await _seed(db)
|
||||||
|
async with db() as s:
|
||||||
|
out = await ann_svc.bulk_upsert("p1", {"img1": _body(2), "ghost": _body(1)}, s)
|
||||||
|
assert out == {"saved": 1, "missing": ["ghost"]}
|
||||||
|
n = (await s.execute(text("SELECT object_count FROM images WHERE id='img1'"))).scalar()
|
||||||
|
assert n == 2
|
||||||
|
got = await ann_svc.get_annotation("p1", "img1", s)
|
||||||
|
assert len(got["shapes"]) == 2
|
||||||
|
st = (await s.execute(text("SELECT status FROM images WHERE id='img1'"))).scalar()
|
||||||
|
assert st == "pending_review"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bulk_upsert_unknown_project_404(db):
|
||||||
|
from fastapi import HTTPException
|
||||||
|
await _seed(db)
|
||||||
|
async with db() as s:
|
||||||
|
with pytest.raises(HTTPException) as e:
|
||||||
|
await ann_svc.bulk_upsert("nope", {"img1": _body()}, s)
|
||||||
|
assert e.value.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_batch_presign_unique_keys_and_cap(db):
|
||||||
|
await _seed(db)
|
||||||
|
async with db() as s:
|
||||||
|
files = [{"fileName": "same.jpg", "contentType": "image/jpeg"} for _ in range(3)]
|
||||||
|
out = await img_svc.batch_presign("p1", files, s)
|
||||||
|
keys = [o["r2Key"] for o in out]
|
||||||
|
assert len(set(keys)) == 3 and all("same.jpg" in k for k in keys)
|
||||||
|
assert all(o["putUrl"] for o in out)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_batch_presign_unknown_project_404(db):
|
||||||
|
from fastapi import HTTPException
|
||||||
|
await _seed(db)
|
||||||
|
async with db() as s:
|
||||||
|
with pytest.raises(HTTPException) as e:
|
||||||
|
await img_svc.batch_presign("nope", [{"fileName": "a.jpg"}], s)
|
||||||
|
assert e.value.status_code == 404
|
||||||
59
tests/test_imports.py
Normal file
59
tests/test_imports.py
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
"""Server zip import: pure COCO mapping + job validation (no R2 touched)."""
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
|
from src.services.imports import build_bulk_from_coco
|
||||||
|
from src.services import imports as svc
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture
|
||||||
|
async def db():
|
||||||
|
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||||
|
from src.core.db import Base
|
||||||
|
async with engine.begin() as conn:
|
||||||
|
await conn.run_sync(Base.metadata.create_all)
|
||||||
|
yield async_sessionmaker(engine, expire_on_commit=False)
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
COCO = {
|
||||||
|
"images": [{"id": 1, "file_name": "a.jpg"}, {"id": 2, "file_name": "b.jpg"}],
|
||||||
|
"annotations": [
|
||||||
|
{"image_id": 1, "category_id": 1, "bbox": [10, 20, 30, 40]},
|
||||||
|
{"image_id": 1, "category_id": 2,
|
||||||
|
"segmentation": [[0, 0, 10, 0, 10, 10, 0, 10]]},
|
||||||
|
{"image_id": 2, "category_id": 9, "iscrowd": 1,
|
||||||
|
"segmentation": {"counts": "xyz"}},
|
||||||
|
{"image_id": 999, "category_id": 1, "bbox": [0, 0, 5, 5]},
|
||||||
|
],
|
||||||
|
"categories": [{"id": 1, "name": "car"}, {"id": 2, "name": "bus"}],
|
||||||
|
}
|
||||||
|
CATS = {"car": {"id": "c1", "name": "car", "color": "#fff"},
|
||||||
|
"bus": {"id": "c2", "name": "bus", "color": "#000"}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_bulk_mapping_bbox_polygon_and_skips():
|
||||||
|
payload = build_bulk_from_coco(
|
||||||
|
COCO, {"a.jpg": "img1", "b.jpg": "img2"}, CATS)
|
||||||
|
assert set(payload) == {"img1"} # b.jpg: RLE-only + unknown image 999 skipped
|
||||||
|
kinds = sorted(s.type for s in payload["img1"].shapes)
|
||||||
|
assert kinds == ["bbox", "polygon"]
|
||||||
|
poly = next(s for s in payload["img1"].shapes if s.type == "polygon")
|
||||||
|
assert [(p.x, p.y) for p in poly.points or []] == [(0, 0), (10, 0), (10, 10), (0, 10)]
|
||||||
|
assert payload["img1"].shapes[0].labelId == "c1"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_start_local_zip_validates(db):
|
||||||
|
from fastapi import HTTPException
|
||||||
|
async with db() as s:
|
||||||
|
await s.execute(text(
|
||||||
|
"INSERT INTO projects (id, slug, name, task_type, status) "
|
||||||
|
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
|
||||||
|
await s.commit()
|
||||||
|
with pytest.raises(HTTPException):
|
||||||
|
await svc.start_local_zip("p1", "relative.zip", s)
|
||||||
|
with pytest.raises(HTTPException):
|
||||||
|
await svc.start_local_zip("nope", "/tmp/x.zip", s)
|
||||||
|
|
@ -54,6 +54,26 @@ async def test_status_and_search_filter_server_side(db):
|
||||||
assert one["total"] == 1 and one["items"][0]["id"] == "img1"
|
assert one["total"] == 1 and one["items"][0]["id"] == "img1"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_locate_image_offset_matches_pages(db):
|
||||||
|
await _seed(db, 5)
|
||||||
|
async with db() as s:
|
||||||
|
loc = await svc.locate_image("p1", "img2", s)
|
||||||
|
assert loc["total"] == 5
|
||||||
|
page = await svc.list_images("p1", "all", s, limit=2, offset=loc["offset"])
|
||||||
|
assert page["items"][0]["id"] == "img2"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_locate_missing_image_is_404(db):
|
||||||
|
from fastapi import HTTPException
|
||||||
|
await _seed(db, 1)
|
||||||
|
async with db() as s:
|
||||||
|
with pytest.raises(HTTPException) as e:
|
||||||
|
await svc.locate_image("p1", "nope", s)
|
||||||
|
assert e.value.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_limit_is_clamped(db):
|
async def test_limit_is_clamped(db):
|
||||||
await _seed(db, 2)
|
await _seed(db, 2)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue