Backend API: auth, projects/images/annotations, R2 presign, tracing, thumbnails, pagination, tests, docs

This commit is contained in:
hailt 2026-11-12 15:12:26 +07:00
parent 5c5deed2f5
commit f8ed08b854
49 changed files with 2626 additions and 0 deletions

0
shared/__init__.py Normal file
View file

121
shared/events.py Normal file
View file

@ -0,0 +1,121 @@
"""Best-effort action-event pipeline (the Orchestrator side).
Hot path: services call emit() — one Redis LPUSH, never raises, never blocks.
A background drainer (started in main lifespan) batch-inserts into action_events.
Redis down -> direct DB insert fallback -> swallowed. Trace data must never
break a user operation.
"""
from __future__ import annotations
import asyncio
import json
import time
from shared.logger import get_logger, session_var, trace_var, user_var
from src.core.config import get_settings
log = get_logger("danable.events")
QUEUE_KEY = "danable:events"
_redis = None
def _redis_client():
global _redis
if _redis is not None:
return _redis
try:
import redis.asyncio as aioredis
_redis = aioredis.from_url(get_settings().redis_url, decode_responses=True,
socket_connect_timeout=1, socket_timeout=1)
except Exception as e:
log.warning("redis unavailable, events fall back to direct insert: %s", e)
_redis = None
return _redis
def _ctx_event(action: str, project_id=None, image_id=None, payload=None,
result=None, latency_ms=None, session_id=None, trace_id=None,
user_id=None) -> dict:
return {
"session_id": session_id or session_var.get() or None,
"trace_id": trace_id or trace_var.get() or None,
"user_id": user_id or user_var.get() or "anonymous",
"action": action,
"project_id": project_id,
"image_id": image_id,
"payload": payload or {},
"result": result,
"latency_ms": latency_ms,
"ts": time.time(),
}
async def _direct_insert(event: dict) -> None:
from models import ActionEvent
from src.core.db import get_session_factory
factory = get_session_factory()
async with factory() as s:
s.add(ActionEvent(session_id=event["session_id"], trace_id=event["trace_id"],
user_id=event["user_id"], action=event["action"],
project_id=event["project_id"], image_id=event["image_id"],
payload=event["payload"], result=event["result"],
latency_ms=event["latency_ms"]))
await s.commit()
async def emit(action: str, **kw) -> None:
"""Enqueue one action event. Never raises."""
try:
event = _ctx_event(action, **kw)
r = _redis_client()
if r is None:
await _direct_insert(event)
return
await r.lpush(QUEUE_KEY, json.dumps(event, ensure_ascii=False))
except Exception as e:
log.warning("emit dropped action=%s: %s", action, e)
async def drain_forever(poll_s: int = 2, batch: int = 200) -> None:
"""Background: Redis queue -> Postgres bulk insert. Cancelled on shutdown."""
from models import ActionEvent
from src.core.db import get_session_factory
log.info("event drainer started")
while True:
try:
r = _redis_client()
if r is None:
await asyncio.sleep(poll_s)
continue
raw = await r.brpop(QUEUE_KEY, timeout=poll_s)
if not raw:
continue
items = [raw[1]]
for _ in range(batch - 1):
extra = await r.rpop(QUEUE_KEY)
if extra is None:
break
items.append(extra)
rows = [ActionEvent(session_id=e.get("session_id"), trace_id=e.get("trace_id"),
user_id=e.get("user_id", "anonymous"), action=e.get("action", "?"),
project_id=e.get("project_id"), image_id=e.get("image_id"),
payload=e.get("payload") or {}, result=e.get("result"),
latency_ms=e.get("latency_ms"))
for e in (json.loads(x) for x in items)]
try:
factory = get_session_factory()
async with factory() as s:
s.add_all(rows)
await s.commit()
except Exception as e:
# DB down: push back so no action is lost, retry next round.
for x in items:
await r.lpush(QUEUE_KEY, x)
raise e
log.info("drained events=%d", len(rows))
except asyncio.CancelledError:
raise
except Exception as e:
log.warning("drainer error: %s", e)
await asyncio.sleep(poll_s)

27
shared/keys.py Normal file
View file

@ -0,0 +1,27 @@
"""R2 key layout. Single source of truth, mirrors config.yaml storage.layout."""
from __future__ import annotations
def image_key(project: str, image_id: str, filename: str) -> str:
safe = filename.replace("/", "_")
return f"datasets/{project}/{image_id}-{safe}"
def thumbnail_key(project: str, image_id: str) -> str:
return f"datasets/{project}/thumbs/{image_id}.jpg"
def thumbnail_small_key(project: str, image_id: str) -> str:
return f"datasets/{project}/thumbs/{image_id}_s.jpg"
def export_key(project: str, filename: str) -> str:
return f"artifacts/exports/{project}/{filename}"
def model_key(name: str, version: str) -> str:
return f"models/{name}/{version}.pt"
def embedding_key(project: str, image_id: str) -> str:
return f"artifacts/embeddings/{project}/{image_id}.npy"

49
shared/logger.py Normal file
View file

@ -0,0 +1,49 @@
"""Central logger: JSON lines carrying session/trace/user context.
Context comes from contextvars (set by shared.tracing per request), so any
service can log and the line is automatically attributable to the user session.
Read with: jq / docker logs. Cold archive: ship JSONL to R2 per day.
"""
from __future__ import annotations
import contextvars
import json
import logging
import sys
from datetime import datetime, timezone
session_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.session_id", default="")
trace_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.trace_id", default="")
user_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.user_id", default="anonymous")
_configured = False
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
payload = {
"ts": datetime.now(timezone.utc).isoformat(),
"level": record.levelname,
"name": record.name,
"session_id": getattr(record, "session_id", None) or session_var.get() or None,
"trace_id": getattr(record, "trace_id", None) or trace_var.get() or None,
"user_id": getattr(record, "user_id", None) or user_var.get() or None,
"msg": record.getMessage(),
}
if record.exc_info:
payload["exc"] = self.formatException(record.exc_info)
return json.dumps(payload, ensure_ascii=False)
def get_logger(name: str = "danable") -> logging.Logger:
global _configured
if not _configured:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger("danable")
root.handlers.clear()
root.addHandler(handler)
root.setLevel(logging.INFO)
root.propagate = False
_configured = True
return logging.getLogger(name)

74
shared/r2.py Normal file
View file

@ -0,0 +1,74 @@
"""Thin boto3 wrapper for R2 (S3-compatible). No image bytes pass through the API."""
from __future__ import annotations
from functools import lru_cache
from src.core.config import get_settings
@lru_cache
def _client():
try:
import boto3
from botocore.config import Config
except ImportError as e:
raise RuntimeError("boto3 not installed: pip install -r requirements.txt") from e
s = get_settings()
if not s.r2_configured:
raise RuntimeError("R2 not configured: set R2_ENDPOINT/R2_ACCESS_KEY_ID/R2_SECRET in .env")
return boto3.client(
"s3",
endpoint_url=s.r2_endpoint,
aws_access_key_id=s.r2_key_id,
aws_secret_access_key=s.r2_secret,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
region_name="auto",
)
def presign_put(key: str, content_type: str = "application/octet-stream") -> str:
"""Direct-upload URL: browser/app PUTs bytes straight to R2 (saves Class-A + bandwidth)."""
s = get_settings()
return _client().generate_presigned_url(
"put_object",
Params={"Bucket": s.r2_bucket, "Key": key, "ContentType": content_type},
ExpiresIn=s.presign_put_ttl,
)
def presign_get(key: str) -> str:
"""Short-lived read URL for preview/download. Thumbnails preferred over full images."""
s = get_settings()
if s.r2_public_base:
return f"{s.r2_public_base.rstrip('/')}/{key}"
return _client().generate_presigned_url(
"get_object",
Params={"Bucket": s.r2_bucket, "Key": key},
ExpiresIn=s.presign_get_ttl,
)
def head_exists(key: str) -> bool:
s = get_settings()
try:
_client().head_object(Bucket=s.r2_bucket, Key=key)
return True
except Exception:
return False
def get_bytes(key: str) -> bytes:
"""Download an object into memory (for workers/thumbnails, not hot paths)."""
s = get_settings()
obj = _client().get_object(Bucket=s.r2_bucket, Key=key)
return obj["Body"].read()
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
cache_control: str | None = None) -> None:
s = get_settings()
params: dict = {"Bucket": s.r2_bucket, "Key": key, "Body": data,
"ContentType": content_type}
if cache_control:
params["CacheControl"] = cache_control
_client().put_object(**params)

75
shared/schemas.py Normal file
View file

@ -0,0 +1,75 @@
"""Pydantic schemas mirroring anotation-frontend/src/types/*."""
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, Field
TaskType = Literal["classification", "object_detection", "segmentation"]
ProjectStatus = Literal["annotating", "pending_review", "completed", "draft"]
ImageStatus = Literal["unassigned", "in_progress", "pending_review", "completed"]
ShapeType = Literal["bbox", "polygon", "point"]
ExportFormat = Literal["coco_json", "yolo_txt", "labelme_json", "mask_png"]
class BBox(BaseModel):
x: float
y: float
width: float
height: float
class Point2D(BaseModel):
x: float
y: float
class AnnotationShape(BaseModel):
id: str
index: int = 0
type: ShapeType
labelId: str
labelName: str = ""
color: str = "#888888"
visible: bool = True
groupId: int | None = None
bbox: BBox | None = None
points: list[Point2D] | None = None
point: Point2D | None = None
class Flags(BaseModel):
blurryOrUncertain: bool = False
needsRecheck: bool = False
class AnnotationUpsert(BaseModel):
classificationLabelId: str | None = None
flags: Flags = Field(default_factory=Flags)
shapes: list[AnnotationShape] = Field(default_factory=list)
class LabelClassIn(BaseModel):
name: str
color: str = "#888888"
shortcutKey: str | None = None
class ProjectCreate(BaseModel):
name: str
slug: str
description: str = ""
taskType: TaskType
class ImageConfirm(BaseModel):
"""After direct PUT to R2, client confirms so API creates metadata."""
fileName: str
r2Key: str
width: int = 0
height: int = 0
class PresignRequest(BaseModel):
fileName: str
contentType: str = "image/jpeg"

64
shared/thumbnails.py Normal file
View file

@ -0,0 +1,64 @@
"""Thumbnail minting: small JPEG covers stored under datasets/{project}/thumbs/.
Pure resize (testable without R2) is separated from R2 IO. Failures never
raise: the card falls back to the full image via _thumbnail_url.
"""
from __future__ import annotations
import asyncio
from io import BytesIO
from shared import keys, r2
from shared.logger import get_logger
from src.core.config import get_settings
log = get_logger("danable.thumbnails")
def resize_to_thumbnail(data: bytes, max_side: int = 512, quality: int = 70) -> bytes:
from PIL import Image as PILImage
with PILImage.open(BytesIO(data)) as im:
im = im.convert("RGB")
im.thumbnail((max_side, max_side), PILImage.LANCZOS)
buf = BytesIO()
im.save(buf, "JPEG", quality=quality, optimize=True)
return buf.getvalue()
# Immutable: keys are unique per image, safe to cache for a year.
IMMUTABLE = "public, max-age=31536000, immutable"
MICRO_SIDE = 128
MICRO_QUALITY = 65
async def mint_thumbnails_for_image(project_id: str, image_id: str,
r2_key: str) -> dict[str, str | None]:
"""Download original -> resize 512 + 128 -> upload both. Never raises."""
s = get_settings()
out: dict[str, str | None] = {"thumbnail": None, "small": None}
if not s.thumbnail_enabled:
return out
try:
original: bytes = await asyncio.to_thread(r2.get_bytes, r2_key)
big = await asyncio.to_thread(
resize_to_thumbnail, original, s.thumbnail_max_side, s.thumbnail_quality)
small = await asyncio.to_thread(
resize_to_thumbnail, original, MICRO_SIDE, MICRO_QUALITY)
thumb_key = keys.thumbnail_key(project_id, image_id)
small_key = keys.thumbnail_small_key(project_id, image_id)
await asyncio.to_thread(r2.put_bytes, thumb_key, big, "image/jpeg", IMMUTABLE)
await asyncio.to_thread(r2.put_bytes, small_key, small, "image/jpeg", IMMUTABLE)
log.info("minted thumbnails %s (%dB) + %s (%dB)",
thumb_key, len(big), small_key, len(small))
out["thumbnail"] = thumb_key
out["small"] = small_key
return out
except Exception as e:
log.warning("thumbnail mint failed key=%s: %s", r2_key, e)
return out
async def mint_thumbnail_for_image(project_id: str, image_id: str, r2_key: str) -> str | None:
"""Legacy single-size wrapper (kept for callers migrating off it)."""
return (await mint_thumbnails_for_image(project_id, image_id, r2_key))["thumbnail"]

70
shared/tracing.py Normal file
View file

@ -0,0 +1,70 @@
"""Request tracing: session_id (one visit) + trace_id (one operation) via contextvars.
Middleware reads X-Session-ID (generated by the client per visit) and X-User-Id,
mints a trace_id per request, and echoes both back as response headers so the
frontend can correlate its own action log with the backend tracelog.
"""
from __future__ import annotations
import time
import uuid
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from shared.logger import get_logger, session_var, trace_var, user_var
log = get_logger("danable.trace")
SESSION_HEADER = "X-Session-ID"
TRACE_HEADER = "X-Trace-ID"
USER_HEADER = "X-User-Id"
def new_id(n: int = 16) -> str:
return uuid.uuid4().hex[:n]
def get_trace_context() -> dict:
return {"session_id": session_var.get(), "trace_id": trace_var.get(),
"user_id": user_var.get()}
def _user_from_token(request) -> str | None:
"""Attribute the trace to the JWT owner without a DB hit."""
try:
from src.core.security import decode_token
auth = request.headers.get("authorization", "")
if not auth.lower().startswith("bearer "):
return None
payload = decode_token(auth.split(" ", 1)[1])
return (payload or {}).get("sub")
except Exception:
return None
class TraceMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
session_id = request.headers.get(SESSION_HEADER, "") or new_id()
trace_id = new_id()
user_id = request.headers.get(USER_HEADER, "") or _user_from_token(request) or "anonymous"
tok_s, tok_t, tok_u = (session_var.set(session_id), trace_var.set(trace_id),
user_var.set(user_id))
start = time.perf_counter()
response = None
try:
response = await call_next(request)
response.headers[SESSION_HEADER] = session_id
response.headers[TRACE_HEADER] = trace_id
return response
except Exception:
log.error("request failed method=%s path=%s", request.method, request.url.path)
raise
finally:
elapsed_ms = (time.perf_counter() - start) * 1000
log.info("request method=%s path=%s status=%s ms=%.1f",
request.method, request.url.path,
response.status_code if response is not None else "err",
elapsed_ms)
session_var.reset(tok_s)
trace_var.reset(tok_t)
user_var.reset(tok_u)

52
shared/traffic.py Normal file
View file

@ -0,0 +1,52 @@
"""Phase-1 traffic layer: in-process per-route counters + slow-request log.
Future (Phase-2, no API change): export these counters to Prometheus/Grafana and
add R2 Class A/B accounting per project. Routers stay untouched.
"""
from __future__ import annotations
import time
from collections import defaultdict
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from shared.logger import get_logger
from src.core.config import get_settings
log = get_logger("danable.traffic")
_counters: dict[str, dict[str, float]] = defaultdict(lambda: {"count": 0, "errors": 0, "total_ms": 0.0})
_started_at = time.time()
def get_snapshot() -> dict:
routes = {
route: {"count": int(v["count"]), "errors": int(v["errors"]),
"avg_ms": round(v["total_ms"] / v["count"], 2) if v["count"] else 0.0}
for route, v in _counters.items()
}
return {"uptime_s": round(time.time() - _started_at, 1), "routes": routes}
class TrafficMiddleware(BaseHTTPMiddleware):
"""Count every request; count 5xx as errors. Zero deps, safe to keep always on."""
async def dispatch(self, request: Request, call_next):
route = f"{request.method} {request.url.path}"
start = time.perf_counter()
try:
response = await call_next(request)
if response.status_code >= 500:
_counters[route]["errors"] += 1
return response
except Exception:
_counters[route]["errors"] += 1
raise
finally:
elapsed_ms = (time.perf_counter() - start) * 1000
_counters[route]["count"] += 1
_counters[route]["total_ms"] += elapsed_ms
slow_ms = get_settings().traffic.get("log_slow_ms", 800)
if elapsed_ms > slow_ms:
log.warning("slow request route=%s ms=%.0f", route, elapsed_ms)