Backend API: auth, projects/images/annotations, R2 presign, tracing, thumbnails, pagination, tests, docs
This commit is contained in:
parent
5c5deed2f5
commit
f8ed08b854
49 changed files with 2626 additions and 0 deletions
0
shared/__init__.py
Normal file
0
shared/__init__.py
Normal file
121
shared/events.py
Normal file
121
shared/events.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
"""Best-effort action-event pipeline (the Orchestrator side).
|
||||
|
||||
Hot path: services call emit() — one Redis LPUSH, never raises, never blocks.
|
||||
A background drainer (started in main lifespan) batch-inserts into action_events.
|
||||
Redis down -> direct DB insert fallback -> swallowed. Trace data must never
|
||||
break a user operation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
|
||||
from shared.logger import get_logger, session_var, trace_var, user_var
|
||||
from src.core.config import get_settings
|
||||
|
||||
log = get_logger("danable.events")
|
||||
QUEUE_KEY = "danable:events"
|
||||
|
||||
_redis = None
|
||||
|
||||
|
||||
def _redis_client():
|
||||
global _redis
|
||||
if _redis is not None:
|
||||
return _redis
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
_redis = aioredis.from_url(get_settings().redis_url, decode_responses=True,
|
||||
socket_connect_timeout=1, socket_timeout=1)
|
||||
except Exception as e:
|
||||
log.warning("redis unavailable, events fall back to direct insert: %s", e)
|
||||
_redis = None
|
||||
return _redis
|
||||
|
||||
|
||||
def _ctx_event(action: str, project_id=None, image_id=None, payload=None,
|
||||
result=None, latency_ms=None, session_id=None, trace_id=None,
|
||||
user_id=None) -> dict:
|
||||
return {
|
||||
"session_id": session_id or session_var.get() or None,
|
||||
"trace_id": trace_id or trace_var.get() or None,
|
||||
"user_id": user_id or user_var.get() or "anonymous",
|
||||
"action": action,
|
||||
"project_id": project_id,
|
||||
"image_id": image_id,
|
||||
"payload": payload or {},
|
||||
"result": result,
|
||||
"latency_ms": latency_ms,
|
||||
"ts": time.time(),
|
||||
}
|
||||
|
||||
|
||||
async def _direct_insert(event: dict) -> None:
|
||||
from models import ActionEvent
|
||||
from src.core.db import get_session_factory
|
||||
factory = get_session_factory()
|
||||
async with factory() as s:
|
||||
s.add(ActionEvent(session_id=event["session_id"], trace_id=event["trace_id"],
|
||||
user_id=event["user_id"], action=event["action"],
|
||||
project_id=event["project_id"], image_id=event["image_id"],
|
||||
payload=event["payload"], result=event["result"],
|
||||
latency_ms=event["latency_ms"]))
|
||||
await s.commit()
|
||||
|
||||
|
||||
async def emit(action: str, **kw) -> None:
|
||||
"""Enqueue one action event. Never raises."""
|
||||
try:
|
||||
event = _ctx_event(action, **kw)
|
||||
r = _redis_client()
|
||||
if r is None:
|
||||
await _direct_insert(event)
|
||||
return
|
||||
await r.lpush(QUEUE_KEY, json.dumps(event, ensure_ascii=False))
|
||||
except Exception as e:
|
||||
log.warning("emit dropped action=%s: %s", action, e)
|
||||
|
||||
|
||||
async def drain_forever(poll_s: int = 2, batch: int = 200) -> None:
|
||||
"""Background: Redis queue -> Postgres bulk insert. Cancelled on shutdown."""
|
||||
from models import ActionEvent
|
||||
from src.core.db import get_session_factory
|
||||
log.info("event drainer started")
|
||||
while True:
|
||||
try:
|
||||
r = _redis_client()
|
||||
if r is None:
|
||||
await asyncio.sleep(poll_s)
|
||||
continue
|
||||
raw = await r.brpop(QUEUE_KEY, timeout=poll_s)
|
||||
if not raw:
|
||||
continue
|
||||
items = [raw[1]]
|
||||
for _ in range(batch - 1):
|
||||
extra = await r.rpop(QUEUE_KEY)
|
||||
if extra is None:
|
||||
break
|
||||
items.append(extra)
|
||||
rows = [ActionEvent(session_id=e.get("session_id"), trace_id=e.get("trace_id"),
|
||||
user_id=e.get("user_id", "anonymous"), action=e.get("action", "?"),
|
||||
project_id=e.get("project_id"), image_id=e.get("image_id"),
|
||||
payload=e.get("payload") or {}, result=e.get("result"),
|
||||
latency_ms=e.get("latency_ms"))
|
||||
for e in (json.loads(x) for x in items)]
|
||||
try:
|
||||
factory = get_session_factory()
|
||||
async with factory() as s:
|
||||
s.add_all(rows)
|
||||
await s.commit()
|
||||
except Exception as e:
|
||||
# DB down: push back so no action is lost, retry next round.
|
||||
for x in items:
|
||||
await r.lpush(QUEUE_KEY, x)
|
||||
raise e
|
||||
log.info("drained events=%d", len(rows))
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
log.warning("drainer error: %s", e)
|
||||
await asyncio.sleep(poll_s)
|
||||
27
shared/keys.py
Normal file
27
shared/keys.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"""R2 key layout. Single source of truth, mirrors config.yaml storage.layout."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def image_key(project: str, image_id: str, filename: str) -> str:
|
||||
safe = filename.replace("/", "_")
|
||||
return f"datasets/{project}/{image_id}-{safe}"
|
||||
|
||||
|
||||
def thumbnail_key(project: str, image_id: str) -> str:
|
||||
return f"datasets/{project}/thumbs/{image_id}.jpg"
|
||||
|
||||
|
||||
def thumbnail_small_key(project: str, image_id: str) -> str:
|
||||
return f"datasets/{project}/thumbs/{image_id}_s.jpg"
|
||||
|
||||
|
||||
def export_key(project: str, filename: str) -> str:
|
||||
return f"artifacts/exports/{project}/{filename}"
|
||||
|
||||
|
||||
def model_key(name: str, version: str) -> str:
|
||||
return f"models/{name}/{version}.pt"
|
||||
|
||||
|
||||
def embedding_key(project: str, image_id: str) -> str:
|
||||
return f"artifacts/embeddings/{project}/{image_id}.npy"
|
||||
49
shared/logger.py
Normal file
49
shared/logger.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""Central logger: JSON lines carrying session/trace/user context.
|
||||
|
||||
Context comes from contextvars (set by shared.tracing per request), so any
|
||||
service can log and the line is automatically attributable to the user session.
|
||||
Read with: jq / docker logs. Cold archive: ship JSONL to R2 per day.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
session_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.session_id", default="")
|
||||
trace_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.trace_id", default="")
|
||||
user_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.user_id", default="anonymous")
|
||||
|
||||
_configured = False
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload = {
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"level": record.levelname,
|
||||
"name": record.name,
|
||||
"session_id": getattr(record, "session_id", None) or session_var.get() or None,
|
||||
"trace_id": getattr(record, "trace_id", None) or trace_var.get() or None,
|
||||
"user_id": getattr(record, "user_id", None) or user_var.get() or None,
|
||||
"msg": record.getMessage(),
|
||||
}
|
||||
if record.exc_info:
|
||||
payload["exc"] = self.formatException(record.exc_info)
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def get_logger(name: str = "danable") -> logging.Logger:
|
||||
global _configured
|
||||
if not _configured:
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root = logging.getLogger("danable")
|
||||
root.handlers.clear()
|
||||
root.addHandler(handler)
|
||||
root.setLevel(logging.INFO)
|
||||
root.propagate = False
|
||||
_configured = True
|
||||
return logging.getLogger(name)
|
||||
74
shared/r2.py
Normal file
74
shared/r2.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Thin boto3 wrapper for R2 (S3-compatible). No image bytes pass through the API."""
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
from src.core.config import get_settings
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _client():
|
||||
try:
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
except ImportError as e:
|
||||
raise RuntimeError("boto3 not installed: pip install -r requirements.txt") from e
|
||||
s = get_settings()
|
||||
if not s.r2_configured:
|
||||
raise RuntimeError("R2 not configured: set R2_ENDPOINT/R2_ACCESS_KEY_ID/R2_SECRET in .env")
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=s.r2_endpoint,
|
||||
aws_access_key_id=s.r2_key_id,
|
||||
aws_secret_access_key=s.r2_secret,
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
region_name="auto",
|
||||
)
|
||||
|
||||
|
||||
def presign_put(key: str, content_type: str = "application/octet-stream") -> str:
|
||||
"""Direct-upload URL: browser/app PUTs bytes straight to R2 (saves Class-A + bandwidth)."""
|
||||
s = get_settings()
|
||||
return _client().generate_presigned_url(
|
||||
"put_object",
|
||||
Params={"Bucket": s.r2_bucket, "Key": key, "ContentType": content_type},
|
||||
ExpiresIn=s.presign_put_ttl,
|
||||
)
|
||||
|
||||
|
||||
def presign_get(key: str) -> str:
|
||||
"""Short-lived read URL for preview/download. Thumbnails preferred over full images."""
|
||||
s = get_settings()
|
||||
if s.r2_public_base:
|
||||
return f"{s.r2_public_base.rstrip('/')}/{key}"
|
||||
return _client().generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": s.r2_bucket, "Key": key},
|
||||
ExpiresIn=s.presign_get_ttl,
|
||||
)
|
||||
|
||||
|
||||
def head_exists(key: str) -> bool:
|
||||
s = get_settings()
|
||||
try:
|
||||
_client().head_object(Bucket=s.r2_bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_bytes(key: str) -> bytes:
|
||||
"""Download an object into memory (for workers/thumbnails, not hot paths)."""
|
||||
s = get_settings()
|
||||
obj = _client().get_object(Bucket=s.r2_bucket, Key=key)
|
||||
return obj["Body"].read()
|
||||
|
||||
|
||||
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
|
||||
cache_control: str | None = None) -> None:
|
||||
s = get_settings()
|
||||
params: dict = {"Bucket": s.r2_bucket, "Key": key, "Body": data,
|
||||
"ContentType": content_type}
|
||||
if cache_control:
|
||||
params["CacheControl"] = cache_control
|
||||
_client().put_object(**params)
|
||||
75
shared/schemas.py
Normal file
75
shared/schemas.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
"""Pydantic schemas mirroring anotation-frontend/src/types/*."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
TaskType = Literal["classification", "object_detection", "segmentation"]
|
||||
ProjectStatus = Literal["annotating", "pending_review", "completed", "draft"]
|
||||
ImageStatus = Literal["unassigned", "in_progress", "pending_review", "completed"]
|
||||
ShapeType = Literal["bbox", "polygon", "point"]
|
||||
ExportFormat = Literal["coco_json", "yolo_txt", "labelme_json", "mask_png"]
|
||||
|
||||
|
||||
class BBox(BaseModel):
|
||||
x: float
|
||||
y: float
|
||||
width: float
|
||||
height: float
|
||||
|
||||
|
||||
class Point2D(BaseModel):
|
||||
x: float
|
||||
y: float
|
||||
|
||||
|
||||
class AnnotationShape(BaseModel):
|
||||
id: str
|
||||
index: int = 0
|
||||
type: ShapeType
|
||||
labelId: str
|
||||
labelName: str = ""
|
||||
color: str = "#888888"
|
||||
visible: bool = True
|
||||
groupId: int | None = None
|
||||
bbox: BBox | None = None
|
||||
points: list[Point2D] | None = None
|
||||
point: Point2D | None = None
|
||||
|
||||
|
||||
class Flags(BaseModel):
|
||||
blurryOrUncertain: bool = False
|
||||
needsRecheck: bool = False
|
||||
|
||||
|
||||
class AnnotationUpsert(BaseModel):
|
||||
classificationLabelId: str | None = None
|
||||
flags: Flags = Field(default_factory=Flags)
|
||||
shapes: list[AnnotationShape] = Field(default_factory=list)
|
||||
|
||||
|
||||
class LabelClassIn(BaseModel):
|
||||
name: str
|
||||
color: str = "#888888"
|
||||
shortcutKey: str | None = None
|
||||
|
||||
|
||||
class ProjectCreate(BaseModel):
|
||||
name: str
|
||||
slug: str
|
||||
description: str = ""
|
||||
taskType: TaskType
|
||||
|
||||
|
||||
class ImageConfirm(BaseModel):
|
||||
"""After direct PUT to R2, client confirms so API creates metadata."""
|
||||
fileName: str
|
||||
r2Key: str
|
||||
width: int = 0
|
||||
height: int = 0
|
||||
|
||||
|
||||
class PresignRequest(BaseModel):
|
||||
fileName: str
|
||||
contentType: str = "image/jpeg"
|
||||
64
shared/thumbnails.py
Normal file
64
shared/thumbnails.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
"""Thumbnail minting: small JPEG covers stored under datasets/{project}/thumbs/.
|
||||
|
||||
Pure resize (testable without R2) is separated from R2 IO. Failures never
|
||||
raise: the card falls back to the full image via _thumbnail_url.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
|
||||
from shared import keys, r2
|
||||
from shared.logger import get_logger
|
||||
from src.core.config import get_settings
|
||||
|
||||
log = get_logger("danable.thumbnails")
|
||||
|
||||
|
||||
def resize_to_thumbnail(data: bytes, max_side: int = 512, quality: int = 70) -> bytes:
|
||||
from PIL import Image as PILImage
|
||||
|
||||
with PILImage.open(BytesIO(data)) as im:
|
||||
im = im.convert("RGB")
|
||||
im.thumbnail((max_side, max_side), PILImage.LANCZOS)
|
||||
buf = BytesIO()
|
||||
im.save(buf, "JPEG", quality=quality, optimize=True)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# Immutable: keys are unique per image, safe to cache for a year.
|
||||
IMMUTABLE = "public, max-age=31536000, immutable"
|
||||
MICRO_SIDE = 128
|
||||
MICRO_QUALITY = 65
|
||||
|
||||
|
||||
async def mint_thumbnails_for_image(project_id: str, image_id: str,
|
||||
r2_key: str) -> dict[str, str | None]:
|
||||
"""Download original -> resize 512 + 128 -> upload both. Never raises."""
|
||||
s = get_settings()
|
||||
out: dict[str, str | None] = {"thumbnail": None, "small": None}
|
||||
if not s.thumbnail_enabled:
|
||||
return out
|
||||
try:
|
||||
original: bytes = await asyncio.to_thread(r2.get_bytes, r2_key)
|
||||
big = await asyncio.to_thread(
|
||||
resize_to_thumbnail, original, s.thumbnail_max_side, s.thumbnail_quality)
|
||||
small = await asyncio.to_thread(
|
||||
resize_to_thumbnail, original, MICRO_SIDE, MICRO_QUALITY)
|
||||
thumb_key = keys.thumbnail_key(project_id, image_id)
|
||||
small_key = keys.thumbnail_small_key(project_id, image_id)
|
||||
await asyncio.to_thread(r2.put_bytes, thumb_key, big, "image/jpeg", IMMUTABLE)
|
||||
await asyncio.to_thread(r2.put_bytes, small_key, small, "image/jpeg", IMMUTABLE)
|
||||
log.info("minted thumbnails %s (%dB) + %s (%dB)",
|
||||
thumb_key, len(big), small_key, len(small))
|
||||
out["thumbnail"] = thumb_key
|
||||
out["small"] = small_key
|
||||
return out
|
||||
except Exception as e:
|
||||
log.warning("thumbnail mint failed key=%s: %s", r2_key, e)
|
||||
return out
|
||||
|
||||
|
||||
async def mint_thumbnail_for_image(project_id: str, image_id: str, r2_key: str) -> str | None:
|
||||
"""Legacy single-size wrapper (kept for callers migrating off it)."""
|
||||
return (await mint_thumbnails_for_image(project_id, image_id, r2_key))["thumbnail"]
|
||||
70
shared/tracing.py
Normal file
70
shared/tracing.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Request tracing: session_id (one visit) + trace_id (one operation) via contextvars.
|
||||
|
||||
Middleware reads X-Session-ID (generated by the client per visit) and X-User-Id,
|
||||
mints a trace_id per request, and echoes both back as response headers so the
|
||||
frontend can correlate its own action log with the backend tracelog.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
|
||||
from shared.logger import get_logger, session_var, trace_var, user_var
|
||||
|
||||
log = get_logger("danable.trace")
|
||||
|
||||
SESSION_HEADER = "X-Session-ID"
|
||||
TRACE_HEADER = "X-Trace-ID"
|
||||
USER_HEADER = "X-User-Id"
|
||||
|
||||
|
||||
def new_id(n: int = 16) -> str:
|
||||
return uuid.uuid4().hex[:n]
|
||||
|
||||
|
||||
def get_trace_context() -> dict:
|
||||
return {"session_id": session_var.get(), "trace_id": trace_var.get(),
|
||||
"user_id": user_var.get()}
|
||||
|
||||
|
||||
def _user_from_token(request) -> str | None:
|
||||
"""Attribute the trace to the JWT owner without a DB hit."""
|
||||
try:
|
||||
from src.core.security import decode_token
|
||||
auth = request.headers.get("authorization", "")
|
||||
if not auth.lower().startswith("bearer "):
|
||||
return None
|
||||
payload = decode_token(auth.split(" ", 1)[1])
|
||||
return (payload or {}).get("sub")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class TraceMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
session_id = request.headers.get(SESSION_HEADER, "") or new_id()
|
||||
trace_id = new_id()
|
||||
user_id = request.headers.get(USER_HEADER, "") or _user_from_token(request) or "anonymous"
|
||||
tok_s, tok_t, tok_u = (session_var.set(session_id), trace_var.set(trace_id),
|
||||
user_var.set(user_id))
|
||||
start = time.perf_counter()
|
||||
response = None
|
||||
try:
|
||||
response = await call_next(request)
|
||||
response.headers[SESSION_HEADER] = session_id
|
||||
response.headers[TRACE_HEADER] = trace_id
|
||||
return response
|
||||
except Exception:
|
||||
log.error("request failed method=%s path=%s", request.method, request.url.path)
|
||||
raise
|
||||
finally:
|
||||
elapsed_ms = (time.perf_counter() - start) * 1000
|
||||
log.info("request method=%s path=%s status=%s ms=%.1f",
|
||||
request.method, request.url.path,
|
||||
response.status_code if response is not None else "err",
|
||||
elapsed_ms)
|
||||
session_var.reset(tok_s)
|
||||
trace_var.reset(tok_t)
|
||||
user_var.reset(tok_u)
|
||||
52
shared/traffic.py
Normal file
52
shared/traffic.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
"""Phase-1 traffic layer: in-process per-route counters + slow-request log.
|
||||
|
||||
Future (Phase-2, no API change): export these counters to Prometheus/Grafana and
|
||||
add R2 Class A/B accounting per project. Routers stay untouched.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
|
||||
from shared.logger import get_logger
|
||||
from src.core.config import get_settings
|
||||
|
||||
log = get_logger("danable.traffic")
|
||||
|
||||
_counters: dict[str, dict[str, float]] = defaultdict(lambda: {"count": 0, "errors": 0, "total_ms": 0.0})
|
||||
_started_at = time.time()
|
||||
|
||||
|
||||
def get_snapshot() -> dict:
|
||||
routes = {
|
||||
route: {"count": int(v["count"]), "errors": int(v["errors"]),
|
||||
"avg_ms": round(v["total_ms"] / v["count"], 2) if v["count"] else 0.0}
|
||||
for route, v in _counters.items()
|
||||
}
|
||||
return {"uptime_s": round(time.time() - _started_at, 1), "routes": routes}
|
||||
|
||||
|
||||
class TrafficMiddleware(BaseHTTPMiddleware):
|
||||
"""Count every request; count 5xx as errors. Zero deps, safe to keep always on."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
route = f"{request.method} {request.url.path}"
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
if response.status_code >= 500:
|
||||
_counters[route]["errors"] += 1
|
||||
return response
|
||||
except Exception:
|
||||
_counters[route]["errors"] += 1
|
||||
raise
|
||||
finally:
|
||||
elapsed_ms = (time.perf_counter() - start) * 1000
|
||||
_counters[route]["count"] += 1
|
||||
_counters[route]["total_ms"] += elapsed_ms
|
||||
slow_ms = get_settings().traffic.get("log_slow_ms", 800)
|
||||
if elapsed_ms > slow_ms:
|
||||
log.warning("slow request route=%s ms=%.0f", route, elapsed_ms)
|
||||
Loading…
Add table
Add a link
Reference in a new issue