feat: add resumable dataset uploads
This commit is contained in:
parent
f8ed08b854
commit
8d40cff227
23 changed files with 913 additions and 21 deletions
84
tests/test_bulk.py
Normal file
84
tests/test_bulk.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Batch upload prep + bulk annotation import (COCO/YOLO path)."""
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from shared.schemas import AnnotationUpsert
|
||||
from src.services import annotations as ann_svc
|
||||
from src.services import images as img_svc
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
from src.core.db import Base
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
yield async_sessionmaker(engine, expire_on_commit=False)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _seed(db):
|
||||
async with db() as s:
|
||||
await s.execute(text(
|
||||
"INSERT INTO projects (id, slug, name, task_type, status) "
|
||||
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
|
||||
await s.execute(text(
|
||||
"INSERT INTO images (id, project_id, file_name, r2_key, status, flags) "
|
||||
"VALUES ('img1', 'p1', 'a.jpg', 'k1', 'in_progress', '{}'),"
|
||||
"('img2', 'p1', 'b.jpg', 'k2', 'in_progress', '{}')"))
|
||||
await s.commit()
|
||||
|
||||
|
||||
def _body(n=1):
|
||||
from shared.schemas import AnnotationShape
|
||||
return AnnotationUpsert(shapes=[
|
||||
AnnotationShape(id=f"s{i}", index=i + 1, type="bbox", labelId="car",
|
||||
bbox={"x": 1, "y": 2, "width": 10, "height": 10})
|
||||
for i in range(n)])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_upsert_saves_and_reports_missing(db):
|
||||
await _seed(db)
|
||||
async with db() as s:
|
||||
out = await ann_svc.bulk_upsert("p1", {"img1": _body(2), "ghost": _body(1)}, s)
|
||||
assert out == {"saved": 1, "missing": ["ghost"]}
|
||||
n = (await s.execute(text("SELECT object_count FROM images WHERE id='img1'"))).scalar()
|
||||
assert n == 2
|
||||
got = await ann_svc.get_annotation("p1", "img1", s)
|
||||
assert len(got["shapes"]) == 2
|
||||
st = (await s.execute(text("SELECT status FROM images WHERE id='img1'"))).scalar()
|
||||
assert st == "pending_review"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_upsert_unknown_project_404(db):
|
||||
from fastapi import HTTPException
|
||||
await _seed(db)
|
||||
async with db() as s:
|
||||
with pytest.raises(HTTPException) as e:
|
||||
await ann_svc.bulk_upsert("nope", {"img1": _body()}, s)
|
||||
assert e.value.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_presign_unique_keys_and_cap(db):
|
||||
await _seed(db)
|
||||
async with db() as s:
|
||||
files = [{"fileName": "same.jpg", "contentType": "image/jpeg"} for _ in range(3)]
|
||||
out = await img_svc.batch_presign("p1", files, s)
|
||||
keys = [o["r2Key"] for o in out]
|
||||
assert len(set(keys)) == 3 and all("same.jpg" in k for k in keys)
|
||||
assert all(o["putUrl"] for o in out)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_presign_unknown_project_404(db):
|
||||
from fastapi import HTTPException
|
||||
await _seed(db)
|
||||
async with db() as s:
|
||||
with pytest.raises(HTTPException) as e:
|
||||
await img_svc.batch_presign("nope", [{"fileName": "a.jpg"}], s)
|
||||
assert e.value.status_code == 404
|
||||
59
tests/test_imports.py
Normal file
59
tests/test_imports.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""Server zip import: pure COCO mapping + job validation (no R2 touched)."""
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from src.services.imports import build_bulk_from_coco
|
||||
from src.services import imports as svc
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
from src.core.db import Base
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
yield async_sessionmaker(engine, expire_on_commit=False)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
COCO = {
|
||||
"images": [{"id": 1, "file_name": "a.jpg"}, {"id": 2, "file_name": "b.jpg"}],
|
||||
"annotations": [
|
||||
{"image_id": 1, "category_id": 1, "bbox": [10, 20, 30, 40]},
|
||||
{"image_id": 1, "category_id": 2,
|
||||
"segmentation": [[0, 0, 10, 0, 10, 10, 0, 10]]},
|
||||
{"image_id": 2, "category_id": 9, "iscrowd": 1,
|
||||
"segmentation": {"counts": "xyz"}},
|
||||
{"image_id": 999, "category_id": 1, "bbox": [0, 0, 5, 5]},
|
||||
],
|
||||
"categories": [{"id": 1, "name": "car"}, {"id": 2, "name": "bus"}],
|
||||
}
|
||||
CATS = {"car": {"id": "c1", "name": "car", "color": "#fff"},
|
||||
"bus": {"id": "c2", "name": "bus", "color": "#000"}}
|
||||
|
||||
|
||||
def test_bulk_mapping_bbox_polygon_and_skips():
|
||||
payload = build_bulk_from_coco(
|
||||
COCO, {"a.jpg": "img1", "b.jpg": "img2"}, CATS)
|
||||
assert set(payload) == {"img1"} # b.jpg: RLE-only + unknown image 999 skipped
|
||||
kinds = sorted(s.type for s in payload["img1"].shapes)
|
||||
assert kinds == ["bbox", "polygon"]
|
||||
poly = next(s for s in payload["img1"].shapes if s.type == "polygon")
|
||||
assert [(p.x, p.y) for p in poly.points or []] == [(0, 0), (10, 0), (10, 10), (0, 10)]
|
||||
assert payload["img1"].shapes[0].labelId == "c1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_local_zip_validates(db):
|
||||
from fastapi import HTTPException
|
||||
async with db() as s:
|
||||
await s.execute(text(
|
||||
"INSERT INTO projects (id, slug, name, task_type, status) "
|
||||
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
|
||||
await s.commit()
|
||||
with pytest.raises(HTTPException):
|
||||
await svc.start_local_zip("p1", "relative.zip", s)
|
||||
with pytest.raises(HTTPException):
|
||||
await svc.start_local_zip("nope", "/tmp/x.zip", s)
|
||||
|
|
@ -54,6 +54,26 @@ async def test_status_and_search_filter_server_side(db):
|
|||
assert one["total"] == 1 and one["items"][0]["id"] == "img1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locate_image_offset_matches_pages(db):
|
||||
await _seed(db, 5)
|
||||
async with db() as s:
|
||||
loc = await svc.locate_image("p1", "img2", s)
|
||||
assert loc["total"] == 5
|
||||
page = await svc.list_images("p1", "all", s, limit=2, offset=loc["offset"])
|
||||
assert page["items"][0]["id"] == "img2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locate_missing_image_is_404(db):
|
||||
from fastapi import HTTPException
|
||||
await _seed(db, 1)
|
||||
async with db() as s:
|
||||
with pytest.raises(HTTPException) as e:
|
||||
await svc.locate_image("p1", "nope", s)
|
||||
assert e.value.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_limit_is_clamped(db):
|
||||
await _seed(db, 2)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue