Backend API: auth, projects/images/annotations, R2 presign, tracing, thumbnails, pagination, tests, docs

This commit is contained in:
hailt 2026-11-12 15:12:26 +07:00
parent 5c5deed2f5
commit f8ed08b854
49 changed files with 2626 additions and 0 deletions

121
shared/events.py Normal file
View file

@ -0,0 +1,121 @@
"""Best-effort action-event pipeline (the Orchestrator side).
Hot path: services call emit() — one Redis LPUSH, never raises, never blocks.
A background drainer (started in main lifespan) batch-inserts into action_events.
Redis down -> direct DB insert fallback -> swallowed. Trace data must never
break a user operation.
"""
from __future__ import annotations
import asyncio
import json
import time
from shared.logger import get_logger, session_var, trace_var, user_var
from src.core.config import get_settings
log = get_logger("danable.events")
QUEUE_KEY = "danable:events"
_redis = None
def _redis_client():
global _redis
if _redis is not None:
return _redis
try:
import redis.asyncio as aioredis
_redis = aioredis.from_url(get_settings().redis_url, decode_responses=True,
socket_connect_timeout=1, socket_timeout=1)
except Exception as e:
log.warning("redis unavailable, events fall back to direct insert: %s", e)
_redis = None
return _redis
def _ctx_event(action: str, project_id=None, image_id=None, payload=None,
result=None, latency_ms=None, session_id=None, trace_id=None,
user_id=None) -> dict:
return {
"session_id": session_id or session_var.get() or None,
"trace_id": trace_id or trace_var.get() or None,
"user_id": user_id or user_var.get() or "anonymous",
"action": action,
"project_id": project_id,
"image_id": image_id,
"payload": payload or {},
"result": result,
"latency_ms": latency_ms,
"ts": time.time(),
}
async def _direct_insert(event: dict) -> None:
from models import ActionEvent
from src.core.db import get_session_factory
factory = get_session_factory()
async with factory() as s:
s.add(ActionEvent(session_id=event["session_id"], trace_id=event["trace_id"],
user_id=event["user_id"], action=event["action"],
project_id=event["project_id"], image_id=event["image_id"],
payload=event["payload"], result=event["result"],
latency_ms=event["latency_ms"]))
await s.commit()
async def emit(action: str, **kw) -> None:
"""Enqueue one action event. Never raises."""
try:
event = _ctx_event(action, **kw)
r = _redis_client()
if r is None:
await _direct_insert(event)
return
await r.lpush(QUEUE_KEY, json.dumps(event, ensure_ascii=False))
except Exception as e:
log.warning("emit dropped action=%s: %s", action, e)
async def drain_forever(poll_s: int = 2, batch: int = 200) -> None:
"""Background: Redis queue -> Postgres bulk insert. Cancelled on shutdown."""
from models import ActionEvent
from src.core.db import get_session_factory
log.info("event drainer started")
while True:
try:
r = _redis_client()
if r is None:
await asyncio.sleep(poll_s)
continue
raw = await r.brpop(QUEUE_KEY, timeout=poll_s)
if not raw:
continue
items = [raw[1]]
for _ in range(batch - 1):
extra = await r.rpop(QUEUE_KEY)
if extra is None:
break
items.append(extra)
rows = [ActionEvent(session_id=e.get("session_id"), trace_id=e.get("trace_id"),
user_id=e.get("user_id", "anonymous"), action=e.get("action", "?"),
project_id=e.get("project_id"), image_id=e.get("image_id"),
payload=e.get("payload") or {}, result=e.get("result"),
latency_ms=e.get("latency_ms"))
for e in (json.loads(x) for x in items)]
try:
factory = get_session_factory()
async with factory() as s:
s.add_all(rows)
await s.commit()
except Exception as e:
# DB down: push back so no action is lost, retry next round.
for x in items:
await r.lpush(QUEUE_KEY, x)
raise e
log.info("drained events=%d", len(rows))
except asyncio.CancelledError:
raise
except Exception as e:
log.warning("drainer error: %s", e)
await asyncio.sleep(poll_s)