Backend API: auth, projects/images/annotations, R2 presign, tracing, thumbnails, pagination, tests, docs

This commit is contained in:
hailt 2026-11-12 15:12:26 +07:00
parent 5c5deed2f5
commit f8ed08b854
49 changed files with 2626 additions and 0 deletions

22
.env.example Normal file
View file

@ -0,0 +1,22 @@
# Copy to .env and fill secrets. NEVER commit .env.
# R2 = S3-compatible. Endpoint bucket-style:
# https://<account_id>.r2.cloudflarestorage.com
APP_ENV=dev
API_HOST=0.0.0.0
API_PORT=8000
# --- Postgres / Redis (docker-compose provides defaults) ---
DATABASE_URL=postgresql+asyncpg://danable:danable@localhost:5432/danable
REDIS_URL=redis://localhost:6379/0
# --- Auth ---
JWT_SECRET=change-me
JWT_EXPIRE_MINUTES=720
# --- Cloudflare R2 (secret part; non-secret shape lives in config.yaml) ---
R2_ENDPOINT=https://651cefb20bd66d84bd12cb62773b5a97.r2.cloudflarestorage.com
R2_ACCESS_KEY_ID=
R2_SECRET_ACCESS_KEY=
R2_BUCKET=danagble
# Optional public/CDN host for direct read links. Empty = presigned URLs only.
R2_PUBLIC_BASE_URL=

223
.gitignore vendored Normal file
View file

@ -0,0 +1,223 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[codz]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py.cover
*.lcov
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
# Pipfile.lock
# uv
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
# poetry.lock
# poetry.toml
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
# pdm.lock
# pdm.toml
.pdm-python
.pdm-build/
# pixi
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
# pixi.lock
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
# in the .venv directory. It is recommended not to include this directory in version control.
.pixi/*
!.pixi/config.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule*
celerybeat.pid
# Redis
*.rdb
*.aof
*.pid
# RabbitMQ
mnesia/
rabbitmq/
rabbitmq-data/
# ActiveMQ
activemq-data/
# SageMath parsed files
*.sage.py
# Environments
.env
.envrc
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Conda environments
conda-meta/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs/Zensical documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
# .idea/
# Abstra
# Abstra is an AI-powered process automation framework.
# Ignore directories containing user credentials, local state, and settings.
# Learn more at https://abstra.io/docs
.abstra/
# Visual Studio Code
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore that
# can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
# and can be added to the global gitignore or merged into this file. However, if you prefer, you
# could uncomment the following to ignore the entire vscode folder
# .vscode/
# Temporary file for partial code execution
tempCodeRunnerFile.py
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
# Marimo
marimo/_static/
marimo/_lsp/
__marimo__/
# Streamlit
.streamlit/secrets.toml

6
Dockerfile Normal file
View file

@ -0,0 +1,6 @@
FROM python:3.11-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"]

133
README.md Normal file
View file

@ -0,0 +1,133 @@
# danable backend
Backend API for danable, a computer-vision data labeling platform
(classification, object detection, segmentation). Metadata lives in
PostgreSQL; image bytes, datasets, models and exports live in Cloudflare R2
(S3-compatible). A Redis-backed action-event pipeline records every user
operation for tracelog and replay.
## Tech stack
- Python 3.10+, FastAPI, SQLAlchemy 2 (asyncio) + asyncpg, Alembic (reserved)
- PostgreSQL 16, Redis 7, Celery (reserved for heavy workers)
- boto3 for R2 (S3-compatible), PyJWT + bcrypt for auth, Pillow for thumbnails
- pytest + pytest-asyncio + aiosqlite for tests
## Quickstart
```bash
cd backend
python3 -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
cp .env.example .env # then fill R2 keys (see Configuration)
docker compose up -d postgres redis
uvicorn src.main:app --host 0.0.0.0 --port 8000 --reload
curl -X POST localhost:8000/api/v1/admin/init-db
```
`init-db` creates all tables and seeds the dev admin:
- email: `admin@danable.local`
- password: `Admin123!`
Change or remove this account before any shared deployment. API docs (Swagger):
`http://localhost:8000/docs`.
Detached runs (servers survive the shell):
```bash
bash restart_api.sh # kill + start API on :8000, health-checks itself
```
## Configuration
Secrets live ONLY in `.env` (gitignored). Everything structural lives in
`config.yaml`. The application never reads secrets from yaml, and never reads
shape/thresholds from env.
| `.env` key | Purpose | Default |
|---|---|---|
| `DATABASE_URL` | Postgres DSN (asyncpg) | `postgresql+asyncpg://danable:danable@localhost:5432/danable` |
| `REDIS_URL` | Redis for the event queue | `redis://localhost:6379/0` |
| `JWT_SECRET` | Signs access/refresh tokens | `change-me` (must override) |
| `JWT_EXPIRE_MINUTES` | Access token lifetime | `720` |
| `R2_ENDPOINT` | `https://<account>.r2.cloudflarestorage.com` | account endpoint |
| `R2_ACCESS_KEY_ID` / `R2_SECRET_ACCESS_KEY` | R2 API token | empty (R2 calls return 503 until set) |
| `R2_BUCKET` | Main private bucket | `danagble` |
| `R2_PUBLIC_BASE_URL` | Optional public/CDN base for reads | empty (presigned URLs only) |
`config.yaml` holds: API prefix and page sizes, R2 key layout
(`datasets/{project}/...`, `models/...`, `artifacts/...`), presigned TTLs,
per-task settings, the AI model registry, traffic budgets, and thumbnail
settings (`enabled`, `max_side: 512`, `quality: 70`, micro `128px`).
## Project structure
```text
backend/
config.yaml / config.example.yaml # non-secret shape (see above)
docker-compose.yml # postgres + redis + api
requirements.txt
models/ # SQLAlchemy tables (Postgres metadata only)
shared/ # cross-cutting: logger, tracing, traffic, r2, keys,
# thumbnails, events, schemas
src/
main.py # app wiring, middleware, lifespan (event drainer)
core/ # config (.env + yaml), db engine, security (JWT/bcrypt)
api/v1/ # thin routers: auth, users, projects, images,
# annotations, events, activities, system
services/ # business logic (routers never touch the DB directly)
stores/ # local dev mirror of the R2 layout (see stores/README.md)
tests/ # pytest suite (sqlite-backed, no external services)
```
Layering rule: routers parse/validate and call `src/services/*`; shared code
never imports routers. Background work goes through the Redis event queue,
never inline in requests.
## Usage workflows
- Label flow: `POST /projects` -> `POST .../images/presign` -> PUT bytes
straight to R2 -> `POST .../images` (confirm mints 512px + 128px thumbs)
-> `PUT .../annotation` per image -> `PATCH .../images/{id}?status=`.
- Image lists are paginated (`limit` <= 200, `offset`, `total` in response)
with server-side `status` / `search` / `assignee` filters. The UI never
fetches the whole dataset.
- Backfill thumbnails for pre-existing rows:
`POST /api/v1/admin/backfill-thumbnails?limit=100`.
- Every mutating call emits an action event (Redis -> `action_events`
table); the activity feed reads that table. Client-only actions arrive via
`POST /api/v1/events` with a `session_id`.
- Tracing: send `X-Session-ID` (one per visit) and the `Authorization` header;
every response echoes `X-Session-ID` / `X-Trace-ID` and JSON logs carry
`session_id`, `trace_id`, `user_id` for replay.
Full endpoint reference: [docs/API.md](docs/API.md).
## Testing
```bash
.venv/bin/python -m pytest tests/ -q
```
Tests are sqlite-backed and need no running services. Current coverage:
auth (hash/token/route protection), annotations (server defaults, 404),
tracing middleware + event ingest, thumbnail resize bounds, pagination
(totals, filters, limit clamp).
## Notes and limits
- `POST /admin/*` endpoints are dev-only: protect or remove them in any
shared deployment.
- Thumbnails are minted inline at confirm time (fail-open; the UI falls back
to the full image). Move to a worker when uploads spike.
- R2 free tier is account-wide: watch Class A/B request counts once many
annotators browse thousand-image projects; the UI paginates (50/page) and
caches immutable thumbs for a year to stay inside it.
## License
Apache-2.0, see [LICENSE](LICENSE).

59
config.example.yaml Normal file
View file

@ -0,0 +1,59 @@
# Non-secret backend configuration. Secrets ONLY in .env.
# Rule: .env decides WHO/WHERE (credentials, hosts), yaml decides HOW (layout, ttl, thresholds).
app:
name: danable
api_prefix: /api/v1
api:
default_page_size: 50
max_page_size: 200
upload_max_mb: 50
database:
pool_size: 10
max_overflow: 10
storage:
bucket_default: danagble
# R2 key layout. {project} = project slug, {image} = image id.
layout:
image: "datasets/{project}/{image}-{filename}"
thumbnail: "datasets/{project}/thumbs/{image}.jpg"
export: "artifacts/exports/{project}/{file}"
model: "models/{name}/{version}.pt"
embedding: "artifacts/embeddings/{project}/{image}.npy"
presigned_ttl:
put_seconds: 900 # upload direct-to-R2
get_seconds: 3600 # preview/download
export_seconds: 86400
# Free-tier guardrails: small thumbs, embedding cache, no proxy of full images.
thumbnail:
enabled: true
max_side: 512
quality: 70
tasks:
classification: { version: 1 }
object_detection: { version: 1 }
segmentation: { version: 1 }
# voice: reserved for later (audio schema + sampler)
ai:
enabled: true
# registry only; heavy weights live on worker hosts, never in git.
models:
- { name: sam-vit-b, kind: segment, device: cpu, max_side: 1024, default: true }
- { name: yolo-detect-m, kind: detect, device: cpu, conf_min: 0.35 }
- { name: clf-resnet50, kind: classify, device: cpu }
cache_embeddings: true # SAM image embedding reuse across point/box prompts
# Placeholder for the future traffic/monitoring layer (see src/middlewares/traffic.py).
# Phase 1: in-process counters only. Phase 2: Prometheus + Grafana, R2 request accounting.
traffic:
enabled: true
log_slow_ms: 800
# Per-route R2 op budget alerts (Class A writes / Class B reads), enforced later.
r2_budgets: { class_a_warn: 800000, class_b_warn: 8000000 }
export:
formats: [coco_json, yolo_txt, labelme_json, mask_png]

59
config.yaml Normal file
View file

@ -0,0 +1,59 @@
# Non-secret backend configuration. Secrets ONLY in .env.
# Rule: .env decides WHO/WHERE (credentials, hosts), yaml decides HOW (layout, ttl, thresholds).
app:
name: danable
api_prefix: /api/v1
api:
default_page_size: 50
max_page_size: 200
upload_max_mb: 50
database:
pool_size: 10
max_overflow: 10
storage:
bucket_default: danagble
# R2 key layout. {project} = project slug, {image} = image id.
layout:
image: "datasets/{project}/{image}-{filename}"
thumbnail: "datasets/{project}/thumbs/{image}.jpg"
export: "artifacts/exports/{project}/{file}"
model: "models/{name}/{version}.pt"
embedding: "artifacts/embeddings/{project}/{image}.npy"
presigned_ttl:
put_seconds: 900 # upload direct-to-R2
get_seconds: 3600 # preview/download
export_seconds: 86400
# Free-tier guardrails: small thumbs, embedding cache, no proxy of full images.
thumbnail:
enabled: true
max_side: 512
quality: 70
tasks:
classification: { version: 1 }
object_detection: { version: 1 }
segmentation: { version: 1 }
# voice: reserved for later (audio schema + sampler)
ai:
enabled: true
# registry only; heavy weights live on worker hosts, never in git.
models:
- { name: sam-vit-b, kind: segment, device: cpu, max_side: 1024, default: true }
- { name: yolo-detect-m, kind: detect, device: cpu, conf_min: 0.35 }
- { name: clf-resnet50, kind: classify, device: cpu }
cache_embeddings: true # SAM image embedding reuse across point/box prompts
# Placeholder for the future traffic/monitoring layer (see src/middlewares/traffic.py).
# Phase 1: in-process counters only. Phase 2: Prometheus + Grafana, R2 request accounting.
traffic:
enabled: true
log_slow_ms: 800
# Per-route R2 op budget alerts (Class A writes / Class B reads), enforced later.
r2_budgets: { class_a_warn: 800000, class_b_warn: 8000000 }
export:
formats: [coco_json, yolo_txt, labelme_json, mask_png]

26
docker-compose.yml Normal file
View file

@ -0,0 +1,26 @@
services:
postgres:
image: postgres:16-alpine
environment:
POSTGRES_USER: danable
POSTGRES_PASSWORD: danable
POSTGRES_DB: danable
ports: ["5432:5432"]
volumes: [pgdata:/var/lib/postgresql/data]
redis:
image: redis:7-alpine
ports: ["6379:6379"]
api:
build: .
env_file: .env
environment:
DATABASE_URL: postgresql+asyncpg://danable:danable@postgres:5432/danable
REDIS_URL: redis://redis:6379/0
ports: ["8000:8000"]
depends_on: [postgres, redis]
command: uvicorn src.main:app --host 0.0.0.0 --port 8000 --reload
volumes:
pgdata: {}

159
docs/API.md Normal file
View file

@ -0,0 +1,159 @@
# danable API reference
Base URL: `http://localhost:8000/api/v1` (prefix from `config.yaml` `app.api_prefix`).
Interactive docs: `http://localhost:8000/docs`.
## Conventions
- Auth: `Authorization: Bearer <accessToken>` on every endpoint except those
marked Public. `GET /auth/login` with wrong credentials returns 401 and the
frontend redirects to `/login`.
- Tracing headers (optional in, always out): send `X-Session-ID` (one ID per
browser visit; server mints one if absent) and optionally `X-User-Id`.
Responses echo `X-Session-ID` and a per-request `X-Trace-ID`.
- Errors are JSON `{ detail: string }`. Common codes: `400` bad input,
`401` missing/invalid token, `404` unknown id, `409` slug/email conflict,
`422` validation, `503` R2 not configured (`.env` keys missing).
- Enums: `taskType`: `classification | object_detection | segmentation`;
image `status`: `unassigned | in_progress | pending_review | completed`;
shape `type`: `bbox | polygon | point`.
## System (Public)
| Method | Path | Description |
|---|---|---|
| GET | `/health` | Liveness incl. DB check, `{ ok: true }` |
| GET | `/metrics` | Phase-1 traffic counters per route |
| GET | `/` (root, no prefix) | Name, docs and health links |
| POST | `/admin/init-db` | Create tables, seed dev admin. Dev-only |
| POST | `/admin/backfill-thumbnails?limit=100` | Mint missing thumbs. Dev-only |
## Auth (Public)
`POST /auth/login` — body `{ email, password }` — returns:
```json
{
"user": { "id": "...", "name": "Admin", "email": "admin@danable.local",
"avatarInitials": "A", "avatarColor": "#3B82F6",
"role": "admin", "status": "active",
"isOnline": true, "lastActive": "..." },
"accessToken": "<jwt ~12h>",
"refreshToken": "<jwt 7d>"
}
```
`GET /auth/me` (auth) — current user in the same shape.
## Users (auth)
| Method | Path | Body | Description |
|---|---|---|---|
| GET | `/users` | — | All users `{ id, email, name, role, status, assignedProjectIds }` |
| POST | `/users` | `{ email, role, projectIds? }` | Invite (unusable password until set), `201` |
| PATCH | `/users/{id}` | `{ role?, status?, projectIds? }` | Role / suspend / project assignment |
Roles: `admin | manager | annotator | reviewer`. Status:
`active | invited | suspended` (suspended tokens are rejected).
## Projects (auth)
| Method | Path | Description |
|---|---|---|
| GET | `/projects` | Full project cards: `stats`, `classes`, `members`, `thumbnailUrl` |
| POST | `/projects` | `{ name, slug, description?, taskType }`, `201` |
| GET | `/projects/{id}` | Detail (same card shape) |
| PATCH | `/projects/{id}` | `{ name?, slug?, description?, task_type?, status? }` |
| DELETE | `/projects/{id}` | Cascades images, annotations, classes, exports, AI and action rows |
| POST | `/projects/{id}/classes` | `{ name, color?, shortcutKey? }`, `201` |
| PUT | `/projects/{id}/classes` | Bulk replace `[{ id?, name, color }]` (matches LabelsTab semantics) |
`stats` example: `{ totalImages, unassigned, inProgress, pendingReview,
completed, totalObjects }`. `members` are users assigned to the project with
per-member `assignedCount`.
## Images (auth)
Direct-to-R2 upload keeps bytes and bandwidth off the API:
1. `POST /projects/{id}/images/presign` `{ fileName, contentType }` →
`{ putUrl, r2Key }` (900s TTL).
2. `PUT <putUrl>` with the file bytes (`Content-Type` must match).
3. `POST /projects/{id}/images` `{ fileName, r2Key, width?, height? }` →
`201 { id, r2Key, url }`. Confirm verifies the object exists, writes the
metadata row and mints 512px + 128px thumbnails (fail-open).
`GET /projects/{id}/images` is paginated — never the whole dataset:
```text
GET /projects/{id}/images?status=all&search=img_&assignee=all&limit=50&offset=0
→ { "items": [ ... ], "total": 1000 }
```
`limit` clamps to 1..200 (config `max_page_size`). Item fields: `id`,
`projectId`, `fileName`, `url`, `thumbnailUrl` (512px), `thumbnailSmallUrl`
(128px, falls back up the chain), `width/height`, `status`, `objectCount`,
`assignedTo/assignedToName`, `reviewerName`, `classificationLabelId`, `flags`.
`PATCH /projects/{id}/images/{imageId}?status=completed` — status transition.
## Annotations (auth)
`GET /projects/{pid}/images/{iid}/annotation` — one row per image:
```json
{ "imageId": "...", "projectId": "...", "classificationLabelId": "cls-1",
"flags": { "blurryOrUncertain": false, "needsRecheck": false },
"shapes": [{ "id": "s1", "index": 1, "type": "bbox",
"labelId": "cls-1", "labelName": "car", "color": "#ff0000",
"visible": true, "bbox": { "x": 10, "y": 20, "width": 100, "height": 80 } }] }
```
`PUT` same path with `{ classificationLabelId?, flags?, shapes[] }` upserts
(one row per image, 404 when the image does not exist) and syncs
`object_count` / flags onto the image row. Also emits an `annotation.save`
action event. Polygon shapes use `points: [{x, y}]`, points use `point`.
## Events and activities
`POST /events` (Public, `202`) — client-side actions the backend never sees:
```json
{ "session_id": "…", "events": [
{ "action": "image.status", "project_id": "…", "image_id": "…",
"payload": { "from": "a", "to": "b" } } ] }
```
`session_id` is required; max 200 events/call. Enqueued to Redis and drained
into `action_events` (requeued on DB failure, never lost, never blocks).
`GET /activities?limit=30` (auth) — human feed derived from that table:
`[{ id, timestamp, userName, actionText, projectId }]`.
## Worked example (curl)
```bash
B=http://localhost:8000/api/v1
curl -s -X POST $B/admin/init-db
TOK=$(curl -s -X POST $B/auth/login -H 'Content-Type: application/json' \
-d '{"email":"admin@danable.local","password":"Admin123!"}' \
| python3 -c "import sys,json; print(json.load(sys.stdin)['accessToken'])")
H="Authorization: Bearer $TOK"
PID=$(curl -s -X POST $B/projects -H "$H" -H 'Content-Type: application/json' \
-d '{"name":"Pho Co","slug":"pho-co","description":"e2e","taskType":"classification"}' \
| python3 -c "import sys,json; print(json.load(sys.stdin)['id'])")
# upload one image
PRE=$(curl -s -X POST $B/projects/$PID/images/presign -H "$H" \
-H 'Content-Type: application/json' -d '{"fileName":"s.jpg","contentType":"image/jpeg"}')
PUTURL=$(echo "$PRE" | python3 -c "import sys,json; print(json.load(sys.stdin)['putUrl'])")
R2KEY=$(echo "$PRE" | python3 -c "import sys,json; print(json.load(sys.stdin)['r2Key'])")
curl -s -o /dev/null -w "%{http_code}\n" -X PUT -H 'Content-Type: image/jpeg' \
--data-binary @photo.jpg "$PUTURL"
IID=$(curl -s -X POST $B/projects/$PID/images -H "$H" -H 'Content-Type: application/json' \
-d "{\"fileName\":\"s.jpg\",\"r2Key\":\"$R2KEY\",\"width\":1280,\"height\":720}" \
| python3 -c "import sys,json; print(json.load(sys.stdin)['id'])")
# classify + complete
curl -s -X PUT $B/projects/$PID/images/$IID/annotation -H "$H" \
-H 'Content-Type: application/json' -d '{"shapes":[],"flags":{}}'
curl -s -X PATCH "$B/projects/$PID/images/$IID?status=completed" -H "$H"
```

14
models/__init__.py Normal file
View file

@ -0,0 +1,14 @@
"""DB tables. Metadata lives in Postgres; bytes live in R2 (r2_key only)."""
from models.tables import (
AIJob,
ActionEvent,
Annotation,
ExportJob,
Image,
LabelClass,
Project,
User,
)
__all__ = ["AIJob", "ActionEvent", "Annotation", "ExportJob", "Image",
"LabelClass", "Project", "User"]

129
models/tables.py Normal file
View file

@ -0,0 +1,129 @@
"""SQLAlchemy models. Metadata lives in Postgres; bytes live in R2 (r2_key only)."""
from __future__ import annotations
import uuid
from datetime import datetime
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.core.db import Base
def _uid() -> str:
return uuid.uuid4().hex[:12]
class User(Base):
__tablename__ = "users"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
email: Mapped[str] = mapped_column(String(255), unique=True, index=True)
name: Mapped[str] = mapped_column(String(255))
role: Mapped[str] = mapped_column(String(32), default="annotator") # admin|manager|annotator|reviewer
status: Mapped[str] = mapped_column(String(32), default="active")
password_hash: Mapped[str] = mapped_column(String(255), default="", server_default="")
assigned_project_ids: Mapped[list] = mapped_column(JSON, default=list)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class Project(Base):
__tablename__ = "projects"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
slug: Mapped[str] = mapped_column(String(255), unique=True, index=True)
name: Mapped[str] = mapped_column(String(255))
description: Mapped[str] = mapped_column(Text, default="", server_default="")
# classification | object_detection | segmentation (+ audio later, no schema change)
task_type: Mapped[str] = mapped_column(String(32), index=True)
status: Mapped[str] = mapped_column(String(32), default="draft")
thumbnail_r2_key: Mapped[str | None] = mapped_column(String(1024), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
classes: Mapped[list["LabelClass"]] = relationship(back_populates="project", cascade="all, delete-orphan")
images: Mapped[list["Image"]] = relationship(back_populates="project", cascade="all, delete-orphan")
class LabelClass(Base):
__tablename__ = "label_classes"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
project_id: Mapped[str] = mapped_column(ForeignKey("projects.id", ondelete="CASCADE"), index=True)
name: Mapped[str] = mapped_column(String(255))
color: Mapped[str] = mapped_column(String(32), default="#888888")
shortcut_key: Mapped[str | None] = mapped_column(String(8), nullable=True)
project: Mapped[Project] = relationship(back_populates="classes")
class Image(Base):
__tablename__ = "images"
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=_uid)
project_id: Mapped[str] = mapped_column(ForeignKey("projects.id", ondelete="CASCADE"), index=True)
file_name: Mapped[str] = mapped_column(String(1024))
r2_key: Mapped[str] = mapped_column(String(1024))
thumbnail_r2_key: Mapped[str | None] = mapped_column(String(1024), nullable=True)
thumbnail_small_r2_key: Mapped[str | None] = mapped_column(String(1024), nullable=True)
width: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
height: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
# unassigned | in_progress | pending_review | completed (mirrors frontend ImageItem)
status: Mapped[str] = mapped_column(String(32), default="unassigned", index=True)
assigned_to: Mapped[str | None] = mapped_column(String(36), nullable=True)
reviewer_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
object_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
classification_label_id: Mapped[str | None] = mapped_column(String(32), nullable=True)
flags: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
project: Mapped[Project] = relationship(back_populates="images")
class Annotation(Base):
"""One row per image (upsert). shapes = list[AnnotationShape] like the frontend."""
__tablename__ = "annotations"
image_id: Mapped[str] = mapped_column(ForeignKey("images.id", ondelete="CASCADE"), primary_key=True)
project_id: Mapped[str] = mapped_column(String(32), index=True)
classification_label_id: Mapped[str | None] = mapped_column(String(32), nullable=True)
flags: Mapped[dict] = mapped_column(JSON, default=dict)
shapes: Mapped[list] = mapped_column(JSON, default=list)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class ActionEvent(Base):
"""One user operation for tracelog/replay: who did what, on which entity,
with which payload. Written by the background drainer, never the hot path."""
__tablename__ = "action_events"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
session_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
trace_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
user_id: Mapped[str] = mapped_column(String(64), default="anonymous")
action: Mapped[str] = mapped_column(String(64), index=True)
project_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
payload: Mapped[dict] = mapped_column(JSON, default=dict)
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
latency_ms: Mapped[float | None] = mapped_column(nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class ExportJob(Base):
__tablename__ = "exports"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
project_id: Mapped[str] = mapped_column(String(32), index=True)
format: Mapped[str] = mapped_column(String(32)) # coco_json | yolo_txt | labelme_json | mask_png
status: Mapped[str] = mapped_column(String(32), default="queued")
r2_key: Mapped[str | None] = mapped_column(String(1024), nullable=True)
record_count: Mapped[int] = mapped_column(Integer, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class AIJob(Base):
"""Assist queue (SAM segment / auto-detect / classify). Worker fills result later."""
__tablename__ = "ai_jobs"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uid)
project_id: Mapped[str] = mapped_column(String(32), index=True)
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
kind: Mapped[str] = mapped_column(String(32)) # segment | detect | classify
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
params: Mapped[dict] = mapped_column(JSON, default=dict)
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
result_r2_key: Mapped[str | None] = mapped_column(String(1024), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())

19
requirements.txt Normal file
View file

@ -0,0 +1,19 @@
fastapi>=0.115
uvicorn[standard]>=0.30
sqlalchemy[asyncio]>=2.0
asyncpg>=0.29
alembic>=1.13
pydantic>=2.7
pydantic-settings>=2.3
PyYAML>=6.0
python-dotenv>=1.0
boto3>=1.34
redis>=5.0
celery>=5.4
bcrypt>=4.1
PyJWT>=2.8
pillow>=10.0
pytest>=8.0
pytest-asyncio>=1.0
aiosqlite>=0.20
httpx>=0.27

9
restart_api.sh Executable file
View file

@ -0,0 +1,9 @@
#!/bin/bash
# Restart danable API safely: pattern lives in this file, never in the caller cmdline.
pkill -f 'uvicorn src\.main:app' 2>/dev/null
sleep 2
cd /home/hailt/danagble/backend
setsid -f .venv/bin/python -m uvicorn src.main:app --host 0.0.0.0 --port 8000 </dev/null >>/tmp/danable-api.log 2>&1
sleep 4
curl -s -m 5 localhost:8000/api/v1/health
echo

0
shared/__init__.py Normal file
View file

121
shared/events.py Normal file
View file

@ -0,0 +1,121 @@
"""Best-effort action-event pipeline (the Orchestrator side).
Hot path: services call emit() — one Redis LPUSH, never raises, never blocks.
A background drainer (started in main lifespan) batch-inserts into action_events.
Redis down -> direct DB insert fallback -> swallowed. Trace data must never
break a user operation.
"""
from __future__ import annotations
import asyncio
import json
import time
from shared.logger import get_logger, session_var, trace_var, user_var
from src.core.config import get_settings
log = get_logger("danable.events")
QUEUE_KEY = "danable:events"
_redis = None
def _redis_client():
global _redis
if _redis is not None:
return _redis
try:
import redis.asyncio as aioredis
_redis = aioredis.from_url(get_settings().redis_url, decode_responses=True,
socket_connect_timeout=1, socket_timeout=1)
except Exception as e:
log.warning("redis unavailable, events fall back to direct insert: %s", e)
_redis = None
return _redis
def _ctx_event(action: str, project_id=None, image_id=None, payload=None,
result=None, latency_ms=None, session_id=None, trace_id=None,
user_id=None) -> dict:
return {
"session_id": session_id or session_var.get() or None,
"trace_id": trace_id or trace_var.get() or None,
"user_id": user_id or user_var.get() or "anonymous",
"action": action,
"project_id": project_id,
"image_id": image_id,
"payload": payload or {},
"result": result,
"latency_ms": latency_ms,
"ts": time.time(),
}
async def _direct_insert(event: dict) -> None:
from models import ActionEvent
from src.core.db import get_session_factory
factory = get_session_factory()
async with factory() as s:
s.add(ActionEvent(session_id=event["session_id"], trace_id=event["trace_id"],
user_id=event["user_id"], action=event["action"],
project_id=event["project_id"], image_id=event["image_id"],
payload=event["payload"], result=event["result"],
latency_ms=event["latency_ms"]))
await s.commit()
async def emit(action: str, **kw) -> None:
"""Enqueue one action event. Never raises."""
try:
event = _ctx_event(action, **kw)
r = _redis_client()
if r is None:
await _direct_insert(event)
return
await r.lpush(QUEUE_KEY, json.dumps(event, ensure_ascii=False))
except Exception as e:
log.warning("emit dropped action=%s: %s", action, e)
async def drain_forever(poll_s: int = 2, batch: int = 200) -> None:
"""Background: Redis queue -> Postgres bulk insert. Cancelled on shutdown."""
from models import ActionEvent
from src.core.db import get_session_factory
log.info("event drainer started")
while True:
try:
r = _redis_client()
if r is None:
await asyncio.sleep(poll_s)
continue
raw = await r.brpop(QUEUE_KEY, timeout=poll_s)
if not raw:
continue
items = [raw[1]]
for _ in range(batch - 1):
extra = await r.rpop(QUEUE_KEY)
if extra is None:
break
items.append(extra)
rows = [ActionEvent(session_id=e.get("session_id"), trace_id=e.get("trace_id"),
user_id=e.get("user_id", "anonymous"), action=e.get("action", "?"),
project_id=e.get("project_id"), image_id=e.get("image_id"),
payload=e.get("payload") or {}, result=e.get("result"),
latency_ms=e.get("latency_ms"))
for e in (json.loads(x) for x in items)]
try:
factory = get_session_factory()
async with factory() as s:
s.add_all(rows)
await s.commit()
except Exception as e:
# DB down: push back so no action is lost, retry next round.
for x in items:
await r.lpush(QUEUE_KEY, x)
raise e
log.info("drained events=%d", len(rows))
except asyncio.CancelledError:
raise
except Exception as e:
log.warning("drainer error: %s", e)
await asyncio.sleep(poll_s)

27
shared/keys.py Normal file
View file

@ -0,0 +1,27 @@
"""R2 key layout. Single source of truth, mirrors config.yaml storage.layout."""
from __future__ import annotations
def image_key(project: str, image_id: str, filename: str) -> str:
safe = filename.replace("/", "_")
return f"datasets/{project}/{image_id}-{safe}"
def thumbnail_key(project: str, image_id: str) -> str:
return f"datasets/{project}/thumbs/{image_id}.jpg"
def thumbnail_small_key(project: str, image_id: str) -> str:
return f"datasets/{project}/thumbs/{image_id}_s.jpg"
def export_key(project: str, filename: str) -> str:
return f"artifacts/exports/{project}/{filename}"
def model_key(name: str, version: str) -> str:
return f"models/{name}/{version}.pt"
def embedding_key(project: str, image_id: str) -> str:
return f"artifacts/embeddings/{project}/{image_id}.npy"

49
shared/logger.py Normal file
View file

@ -0,0 +1,49 @@
"""Central logger: JSON lines carrying session/trace/user context.
Context comes from contextvars (set by shared.tracing per request), so any
service can log and the line is automatically attributable to the user session.
Read with: jq / docker logs. Cold archive: ship JSONL to R2 per day.
"""
from __future__ import annotations
import contextvars
import json
import logging
import sys
from datetime import datetime, timezone
session_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.session_id", default="")
trace_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.trace_id", default="")
user_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.user_id", default="anonymous")
_configured = False
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
payload = {
"ts": datetime.now(timezone.utc).isoformat(),
"level": record.levelname,
"name": record.name,
"session_id": getattr(record, "session_id", None) or session_var.get() or None,
"trace_id": getattr(record, "trace_id", None) or trace_var.get() or None,
"user_id": getattr(record, "user_id", None) or user_var.get() or None,
"msg": record.getMessage(),
}
if record.exc_info:
payload["exc"] = self.formatException(record.exc_info)
return json.dumps(payload, ensure_ascii=False)
def get_logger(name: str = "danable") -> logging.Logger:
global _configured
if not _configured:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger("danable")
root.handlers.clear()
root.addHandler(handler)
root.setLevel(logging.INFO)
root.propagate = False
_configured = True
return logging.getLogger(name)

74
shared/r2.py Normal file
View file

@ -0,0 +1,74 @@
"""Thin boto3 wrapper for R2 (S3-compatible). No image bytes pass through the API."""
from __future__ import annotations
from functools import lru_cache
from src.core.config import get_settings
@lru_cache
def _client():
try:
import boto3
from botocore.config import Config
except ImportError as e:
raise RuntimeError("boto3 not installed: pip install -r requirements.txt") from e
s = get_settings()
if not s.r2_configured:
raise RuntimeError("R2 not configured: set R2_ENDPOINT/R2_ACCESS_KEY_ID/R2_SECRET in .env")
return boto3.client(
"s3",
endpoint_url=s.r2_endpoint,
aws_access_key_id=s.r2_key_id,
aws_secret_access_key=s.r2_secret,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
region_name="auto",
)
def presign_put(key: str, content_type: str = "application/octet-stream") -> str:
"""Direct-upload URL: browser/app PUTs bytes straight to R2 (saves Class-A + bandwidth)."""
s = get_settings()
return _client().generate_presigned_url(
"put_object",
Params={"Bucket": s.r2_bucket, "Key": key, "ContentType": content_type},
ExpiresIn=s.presign_put_ttl,
)
def presign_get(key: str) -> str:
"""Short-lived read URL for preview/download. Thumbnails preferred over full images."""
s = get_settings()
if s.r2_public_base:
return f"{s.r2_public_base.rstrip('/')}/{key}"
return _client().generate_presigned_url(
"get_object",
Params={"Bucket": s.r2_bucket, "Key": key},
ExpiresIn=s.presign_get_ttl,
)
def head_exists(key: str) -> bool:
s = get_settings()
try:
_client().head_object(Bucket=s.r2_bucket, Key=key)
return True
except Exception:
return False
def get_bytes(key: str) -> bytes:
"""Download an object into memory (for workers/thumbnails, not hot paths)."""
s = get_settings()
obj = _client().get_object(Bucket=s.r2_bucket, Key=key)
return obj["Body"].read()
def put_bytes(key: str, data: bytes, content_type: str = "image/jpeg",
cache_control: str | None = None) -> None:
s = get_settings()
params: dict = {"Bucket": s.r2_bucket, "Key": key, "Body": data,
"ContentType": content_type}
if cache_control:
params["CacheControl"] = cache_control
_client().put_object(**params)

75
shared/schemas.py Normal file
View file

@ -0,0 +1,75 @@
"""Pydantic schemas mirroring anotation-frontend/src/types/*."""
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, Field
TaskType = Literal["classification", "object_detection", "segmentation"]
ProjectStatus = Literal["annotating", "pending_review", "completed", "draft"]
ImageStatus = Literal["unassigned", "in_progress", "pending_review", "completed"]
ShapeType = Literal["bbox", "polygon", "point"]
ExportFormat = Literal["coco_json", "yolo_txt", "labelme_json", "mask_png"]
class BBox(BaseModel):
x: float
y: float
width: float
height: float
class Point2D(BaseModel):
x: float
y: float
class AnnotationShape(BaseModel):
id: str
index: int = 0
type: ShapeType
labelId: str
labelName: str = ""
color: str = "#888888"
visible: bool = True
groupId: int | None = None
bbox: BBox | None = None
points: list[Point2D] | None = None
point: Point2D | None = None
class Flags(BaseModel):
blurryOrUncertain: bool = False
needsRecheck: bool = False
class AnnotationUpsert(BaseModel):
classificationLabelId: str | None = None
flags: Flags = Field(default_factory=Flags)
shapes: list[AnnotationShape] = Field(default_factory=list)
class LabelClassIn(BaseModel):
name: str
color: str = "#888888"
shortcutKey: str | None = None
class ProjectCreate(BaseModel):
name: str
slug: str
description: str = ""
taskType: TaskType
class ImageConfirm(BaseModel):
"""After direct PUT to R2, client confirms so API creates metadata."""
fileName: str
r2Key: str
width: int = 0
height: int = 0
class PresignRequest(BaseModel):
fileName: str
contentType: str = "image/jpeg"

64
shared/thumbnails.py Normal file
View file

@ -0,0 +1,64 @@
"""Thumbnail minting: small JPEG covers stored under datasets/{project}/thumbs/.
Pure resize (testable without R2) is separated from R2 IO. Failures never
raise: the card falls back to the full image via _thumbnail_url.
"""
from __future__ import annotations
import asyncio
from io import BytesIO
from shared import keys, r2
from shared.logger import get_logger
from src.core.config import get_settings
log = get_logger("danable.thumbnails")
def resize_to_thumbnail(data: bytes, max_side: int = 512, quality: int = 70) -> bytes:
from PIL import Image as PILImage
with PILImage.open(BytesIO(data)) as im:
im = im.convert("RGB")
im.thumbnail((max_side, max_side), PILImage.LANCZOS)
buf = BytesIO()
im.save(buf, "JPEG", quality=quality, optimize=True)
return buf.getvalue()
# Immutable: keys are unique per image, safe to cache for a year.
IMMUTABLE = "public, max-age=31536000, immutable"
MICRO_SIDE = 128
MICRO_QUALITY = 65
async def mint_thumbnails_for_image(project_id: str, image_id: str,
r2_key: str) -> dict[str, str | None]:
"""Download original -> resize 512 + 128 -> upload both. Never raises."""
s = get_settings()
out: dict[str, str | None] = {"thumbnail": None, "small": None}
if not s.thumbnail_enabled:
return out
try:
original: bytes = await asyncio.to_thread(r2.get_bytes, r2_key)
big = await asyncio.to_thread(
resize_to_thumbnail, original, s.thumbnail_max_side, s.thumbnail_quality)
small = await asyncio.to_thread(
resize_to_thumbnail, original, MICRO_SIDE, MICRO_QUALITY)
thumb_key = keys.thumbnail_key(project_id, image_id)
small_key = keys.thumbnail_small_key(project_id, image_id)
await asyncio.to_thread(r2.put_bytes, thumb_key, big, "image/jpeg", IMMUTABLE)
await asyncio.to_thread(r2.put_bytes, small_key, small, "image/jpeg", IMMUTABLE)
log.info("minted thumbnails %s (%dB) + %s (%dB)",
thumb_key, len(big), small_key, len(small))
out["thumbnail"] = thumb_key
out["small"] = small_key
return out
except Exception as e:
log.warning("thumbnail mint failed key=%s: %s", r2_key, e)
return out
async def mint_thumbnail_for_image(project_id: str, image_id: str, r2_key: str) -> str | None:
"""Legacy single-size wrapper (kept for callers migrating off it)."""
return (await mint_thumbnails_for_image(project_id, image_id, r2_key))["thumbnail"]

70
shared/tracing.py Normal file
View file

@ -0,0 +1,70 @@
"""Request tracing: session_id (one visit) + trace_id (one operation) via contextvars.
Middleware reads X-Session-ID (generated by the client per visit) and X-User-Id,
mints a trace_id per request, and echoes both back as response headers so the
frontend can correlate its own action log with the backend tracelog.
"""
from __future__ import annotations
import time
import uuid
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from shared.logger import get_logger, session_var, trace_var, user_var
log = get_logger("danable.trace")
SESSION_HEADER = "X-Session-ID"
TRACE_HEADER = "X-Trace-ID"
USER_HEADER = "X-User-Id"
def new_id(n: int = 16) -> str:
return uuid.uuid4().hex[:n]
def get_trace_context() -> dict:
return {"session_id": session_var.get(), "trace_id": trace_var.get(),
"user_id": user_var.get()}
def _user_from_token(request) -> str | None:
"""Attribute the trace to the JWT owner without a DB hit."""
try:
from src.core.security import decode_token
auth = request.headers.get("authorization", "")
if not auth.lower().startswith("bearer "):
return None
payload = decode_token(auth.split(" ", 1)[1])
return (payload or {}).get("sub")
except Exception:
return None
class TraceMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
session_id = request.headers.get(SESSION_HEADER, "") or new_id()
trace_id = new_id()
user_id = request.headers.get(USER_HEADER, "") or _user_from_token(request) or "anonymous"
tok_s, tok_t, tok_u = (session_var.set(session_id), trace_var.set(trace_id),
user_var.set(user_id))
start = time.perf_counter()
response = None
try:
response = await call_next(request)
response.headers[SESSION_HEADER] = session_id
response.headers[TRACE_HEADER] = trace_id
return response
except Exception:
log.error("request failed method=%s path=%s", request.method, request.url.path)
raise
finally:
elapsed_ms = (time.perf_counter() - start) * 1000
log.info("request method=%s path=%s status=%s ms=%.1f",
request.method, request.url.path,
response.status_code if response is not None else "err",
elapsed_ms)
session_var.reset(tok_s)
trace_var.reset(tok_t)
user_var.reset(tok_u)

52
shared/traffic.py Normal file
View file

@ -0,0 +1,52 @@
"""Phase-1 traffic layer: in-process per-route counters + slow-request log.
Future (Phase-2, no API change): export these counters to Prometheus/Grafana and
add R2 Class A/B accounting per project. Routers stay untouched.
"""
from __future__ import annotations
import time
from collections import defaultdict
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from shared.logger import get_logger
from src.core.config import get_settings
log = get_logger("danable.traffic")
_counters: dict[str, dict[str, float]] = defaultdict(lambda: {"count": 0, "errors": 0, "total_ms": 0.0})
_started_at = time.time()
def get_snapshot() -> dict:
routes = {
route: {"count": int(v["count"]), "errors": int(v["errors"]),
"avg_ms": round(v["total_ms"] / v["count"], 2) if v["count"] else 0.0}
for route, v in _counters.items()
}
return {"uptime_s": round(time.time() - _started_at, 1), "routes": routes}
class TrafficMiddleware(BaseHTTPMiddleware):
"""Count every request; count 5xx as errors. Zero deps, safe to keep always on."""
async def dispatch(self, request: Request, call_next):
route = f"{request.method} {request.url.path}"
start = time.perf_counter()
try:
response = await call_next(request)
if response.status_code >= 500:
_counters[route]["errors"] += 1
return response
except Exception:
_counters[route]["errors"] += 1
raise
finally:
elapsed_ms = (time.perf_counter() - start) * 1000
_counters[route]["count"] += 1
_counters[route]["total_ms"] += elapsed_ms
slow_ms = get_settings().traffic.get("log_slow_ms", 800)
if elapsed_ms > slow_ms:
log.warning("slow request route=%s ms=%.0f", route, elapsed_ms)

0
src/__init__.py Normal file
View file

0
src/api/__init__.py Normal file
View file

0
src/api/v1/__init__.py Normal file
View file

40
src/api/v1/activities.py Normal file
View file

@ -0,0 +1,40 @@
"""Activity feed derived from the action-event tracelog (no separate table)."""
from __future__ import annotations
from fastapi import APIRouter, Depends, Query
from sqlalchemy import desc, select
from sqlalchemy.ext.asyncio import AsyncSession
from models import ActionEvent, User
from src.core.db import get_session
router = APIRouter(prefix="/activities", tags=["activities"])
_ACTION_TEXT = {
"project.create": "đã tạo dự án mới",
"image.confirm": "đã tải ảnh mới lên",
"image.status": "đã đổi trạng thái ảnh",
"annotation.save": "đã lưu nhãn ảnh",
"dataset.export": "đã xuất dataset",
"image.next": "đã chuyển ảnh",
}
def _describe(e: ActionEvent, names: dict) -> str:
base = _ACTION_TEXT.get(e.action, f"đã thực hiện {e.action}")
who = names.get(e.user_id or "", None) or (e.user_id or "hệ thống")
detail = e.image_id or e.project_id or ""
return f"{who} {base} {detail}".strip()
@router.get("")
async def list_activities(limit: int = Query(default=30, le=200),
db: AsyncSession = Depends(get_session)):
rows = (await db.execute(select(ActionEvent).order_by(desc(ActionEvent.created_at)).limit(limit))).scalars().all()
users = (await db.execute(select(User))).scalars().all()
names = {u.id: u.name for u in users}
return [{"id": e.id,
"timestamp": e.created_at.isoformat() if e.created_at else "",
"userName": names.get(e.user_id or "", e.user_id or "hệ thống"),
"actionText": _describe(e, names),
"projectId": e.project_id} for e in rows]

21
src/api/v1/annotations.py Normal file
View file

@ -0,0 +1,21 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from shared.schemas import AnnotationUpsert
from src.core.db import get_session
from src.services import annotations as svc
router = APIRouter(prefix="/projects/{project_id}/images/{image_id}/annotation", tags=["annotations"])
@router.get("")
async def get_annotation(project_id: str, image_id: str, db: AsyncSession = Depends(get_session)):
return await svc.get_annotation(project_id, image_id, db)
@router.put("")
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
db: AsyncSession = Depends(get_session)):
return await svc.upsert_annotation(project_id, image_id, body, db)

73
src/api/v1/auth.py Normal file
View file

@ -0,0 +1,73 @@
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models import User
from src.core.db import get_session
from src.core.security import create_token, get_current_user, verify_password
router = APIRouter(prefix="/auth", tags=["auth"])
class LoginIn(BaseModel):
email: str
password: str = ""
@router.post("/login")
async def login(body: LoginIn, db: AsyncSession = Depends(get_session)):
user = (await db.execute(
select(User).where(User.email == body.email.strip().lower()))).scalar_one_or_none()
if user is None or not verify_password(body.password or "", user.password_hash or ""):
raise HTTPException(401, "email hoặc mật khẩu không đúng")
access = create_token(user.id, user.role)
refresh = create_token(user.id, user.role, expires_minutes=60 * 24 * 7)
return {"user": {"id": user.id, "name": user.name, "email": user.email,
"avatarInitials": "".join(p[0] for p in user.name.split()[:2]).upper(),
"avatarColor": "#3B82F6", "role": user.role, "status": user.status,
"isOnline": True, "lastActive": "Đang trực tuyến"},
"accessToken": access, "refreshToken": refresh}
@router.get("/me")
async def me(user: User = Depends(get_current_user)):
return {"id": user.id, "name": user.name, "email": user.email,
"avatarInitials": "".join(p[0] for p in user.name.split()[:2]).upper(),
"avatarColor": "#3B82F6", "role": user.role, "status": user.status,
"isOnline": True, "lastActive": "Đang trực tuyến"}
class InviteIn(BaseModel):
email: str
role: str = "annotator"
projectIds: list[str] = Field(default_factory=list)
class UserPatch(BaseModel):
role: str | None = None
status: str | None = None
projectIds: list[str] | None = None
users_router = APIRouter(prefix="/users", tags=["users"])
@users_router.get("")
async def list_users(db: AsyncSession = Depends(get_session)):
from src.services import users as svc
return await svc.list_users(db)
@users_router.post("", status_code=201)
async def invite(body: InviteIn, db: AsyncSession = Depends(get_session)):
from src.services import users as svc
return await svc.invite_user(body.email, body.role, body.projectIds, db)
@users_router.patch("/{user_id}")
async def patch_user(user_id: str, body: UserPatch, db: AsyncSession = Depends(get_session)):
from src.services import users as svc
return await svc.patch_user(user_id, body.model_dump(), db)

39
src/api/v1/events.py Normal file
View file

@ -0,0 +1,39 @@
"""Client action ingest: operations the backend never sees (UI-only actions).
Batch format keeps high-frequency UI affordable: the frontend buffers and sends
up to 200 events per call. session_id is required — an event without a session
cannot be replayed.
"""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from shared import events
from shared.tracing import new_id
router = APIRouter(prefix="/events", tags=["events"])
class ClientEvent(BaseModel):
action: str = Field(min_length=1, max_length=64)
project_id: str | None = None
image_id: str | None = None
payload: dict = Field(default_factory=dict)
class EventsIn(BaseModel):
session_id: str = Field(min_length=1, max_length=64)
events: list[ClientEvent] = Field(min_length=1, max_length=200)
@router.post("", status_code=202)
async def ingest(body: EventsIn):
if not body.session_id.strip():
raise HTTPException(400, "session_id is required")
for e in body.events:
await events.emit(e.action, project_id=e.project_id, image_id=e.image_id,
payload=e.payload, session_id=body.session_id,
trace_id=new_id())
return {"accepted": len(body.events)}

37
src/api/v1/images.py Normal file
View file

@ -0,0 +1,37 @@
from __future__ import annotations
from fastapi import APIRouter, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from shared.schemas import ImageConfirm, PresignRequest
from src.core.db import get_session
from src.services import images as svc
router = APIRouter(prefix="/projects/{project_id}/images", tags=["images"])
@router.get("")
async def list_images(project_id: str, status: str | None = Query(default=None),
search: str | None = Query(default=None),
assignee: str | None = Query(default=None),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
db: AsyncSession = Depends(get_session)):
return await svc.list_images(project_id, status, db, search=search,
assignee=assignee, limit=limit, offset=offset)
@router.post("/presign")
async def presign_upload(project_id: str, body: PresignRequest):
return svc.build_presign(project_id, body.fileName, body.contentType)
@router.post("", status_code=201)
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession = Depends(get_session)):
return await svc.confirm_image(project_id, body, db)
@router.patch("/{image_id}")
async def update_status(project_id: str, image_id: str, status: str,
db: AsyncSession = Depends(get_session)):
return await svc.update_status(project_id, image_id, status, db)

47
src/api/v1/projects.py Normal file
View file

@ -0,0 +1,47 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from shared.schemas import LabelClassIn, ProjectCreate
from src.core.db import get_session
from src.services import projects as svc
router = APIRouter(prefix="/projects", tags=["projects"])
@router.get("")
async def list_projects(db: AsyncSession = Depends(get_session)):
return await svc.list_projects(db)
@router.post("", status_code=201)
async def create_project(body: ProjectCreate, db: AsyncSession = Depends(get_session)):
return await svc.create_project(body, db)
@router.get("/{project_id}")
async def get_project(project_id: str, db: AsyncSession = Depends(get_session)):
return await svc.get_project_detail(project_id, db)
@router.patch("/{project_id}")
async def patch_project(project_id: str, body: dict, db: AsyncSession = Depends(get_session)):
return await svc.patch_project(project_id, body, db)
@router.delete("/{project_id}")
async def delete_project(project_id: str, db: AsyncSession = Depends(get_session)):
return await svc.delete_project(project_id, db)
@router.post("/{project_id}/classes", status_code=201)
async def add_class(project_id: str, body: LabelClassIn, db: AsyncSession = Depends(get_session)):
return await svc.add_label_class(project_id, body, db)
@router.put("/{project_id}/classes")
async def replace_classes(project_id: str, body: list[dict],
db: AsyncSession = Depends(get_session)):
"""Bulk replace backing LabelsTab onUpdateClasses (add/edit/delete in one call)."""
return await svc.replace_classes(project_id, body, db)

60
src/api/v1/system.py Normal file
View file

@ -0,0 +1,60 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
from shared.traffic import get_snapshot
from src.core.db import Base, get_engine, get_session, get_session_factory
router = APIRouter(tags=["system"])
@router.get("/health")
async def health(db: AsyncSession = Depends(get_session)):
await db.execute(text("SELECT 1"))
return {"ok": True}
@router.get("/metrics")
async def metrics():
"""Phase-1 traffic counters (Phase-2: Prometheus export). No auth on dev; protect in prod."""
return get_snapshot()
@router.post("/admin/backfill-thumbnails")
async def backfill_thumbnails(limit: int = 100, db: AsyncSession = Depends(get_session)):
"""Mint thumbs for images uploaded before thumbnailing existed. Dev-only."""
from models import Image
from shared import thumbnails
rows = (await db.execute(
select(Image).where(Image.thumbnail_r2_key.is_(None)).limit(limit))).scalars().all()
minted = 0
for im in rows:
keys = await thumbnails.mint_thumbnails_for_image(im.project_id, im.id, im.r2_key)
if keys["thumbnail"]:
im.thumbnail_r2_key = keys["thumbnail"]
if keys["small"]:
im.thumbnail_small_r2_key = keys["small"]
if keys["thumbnail"] or keys["small"]:
minted += 1
await db.commit()
return {"scanned": len(rows), "minted": minted}
@router.post("/admin/init-db")
async def init_db():
"""Create tables from models. Replaced by Alembic once migrations exist."""
from models import User
from src.core.security import hash_password
async with get_engine().begin() as conn:
await conn.run_sync(Base.metadata.create_all)
factory = get_session_factory()
async with factory() as s:
admin = (await s.execute(select(User).where(User.email == "admin@danable.local"))).scalar_one_or_none()
if admin is None:
s.add(User(email="admin@danable.local", name="Admin", role="admin",
status="active", password_hash=hash_password("Admin123!"),
assigned_project_ids=[]))
await s.commit()
return {"ok": True, "seeded_admin": "admin@danable.local"}

0
src/core/__init__.py Normal file
View file

65
src/core/config.py Normal file
View file

@ -0,0 +1,65 @@
"""Central settings: .env (secrets) + config.yaml (shape). Env wins on conflict."""
from __future__ import annotations
import os
from functools import lru_cache
from pathlib import Path
import yaml
from dotenv import load_dotenv
BACKEND_DIR = Path(__file__).resolve().parents[2]
load_dotenv(BACKEND_DIR / ".env")
def _load_yaml() -> dict:
for name in ("config.yaml", "config.example.yaml"):
p = BACKEND_DIR / name
if p.exists():
return yaml.safe_load(p.read_text()) or {}
return {}
class Settings:
def __init__(self) -> None:
self.yaml = _load_yaml()
app = self.yaml.get("app", {})
storage = self.yaml.get("storage", {})
self.app_name: str = app.get("name", "danable")
self.api_prefix: str = app.get("api_prefix", "/api/v1")
self.database_url: str = os.getenv(
"DATABASE_URL", "postgresql+asyncpg://danable:danable@localhost:5432/danable"
)
self.redis_url: str = os.getenv("REDIS_URL", "redis://localhost:6379/0")
self.jwt_secret: str = os.getenv("JWT_SECRET", "change-me")
self.jwt_expire_minutes: int = int(os.getenv("JWT_EXPIRE_MINUTES", "720"))
self.r2_endpoint: str = os.getenv("R2_ENDPOINT", "")
self.r2_key_id: str = os.getenv("R2_ACCESS_KEY_ID", "")
self.r2_secret: str = os.getenv("R2_SECRET_ACCESS_KEY", "")
self.r2_bucket: str = os.getenv(
"R2_BUCKET", storage.get("bucket_default", "danagble")
)
self.r2_public_base: str = os.getenv("R2_PUBLIC_BASE_URL", "")
layout = storage.get("layout", {})
presigned = storage.get("presigned_ttl", {})
self.r2_layout: dict = layout
thumb = storage.get("thumbnail", {})
self.thumbnail_enabled: bool = bool(thumb.get("enabled", True))
self.thumbnail_max_side: int = int(thumb.get("max_side", 512))
self.thumbnail_quality: int = int(thumb.get("quality", 70))
self.presign_put_ttl: int = int(presigned.get("put_seconds", 900))
self.presign_get_ttl: int = int(presigned.get("get_seconds", 3600))
self.traffic = self.yaml.get("traffic", {"enabled": True})
@property
def r2_configured(self) -> bool:
return bool(self.r2_endpoint and self.r2_key_id and self.r2_secret)
@lru_cache
def get_settings() -> Settings:
return Settings()

36
src/core/db.py Normal file
View file

@ -0,0 +1,36 @@
"""Async SQLAlchemy engine/session. Postgres only (per project decision)."""
from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase
from src.core.config import get_settings
class Base(DeclarativeBase):
pass
_engine = None
_session_factory = None
def get_engine():
global _engine
if _engine is None:
s = get_settings()
_engine = create_async_engine(s.database_url, pool_pre_ping=True)
return _engine
def get_session_factory() -> async_sessionmaker[AsyncSession]:
global _session_factory
if _session_factory is None:
_session_factory = async_sessionmaker(get_engine(), expire_on_commit=False)
return _session_factory
async def get_session():
factory = get_session_factory()
async with factory() as session:
yield session

63
src/core/security.py Normal file
View file

@ -0,0 +1,63 @@
"""JWT + bcrypt auth. Passwords never leave the hash; tokens carry sub/role/exp."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import bcrypt
import jwt
from fastapi import Depends, HTTPException, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy.ext.asyncio import AsyncSession
from src.core.config import get_settings
from src.core.db import get_session
_bearer = HTTPBearer(auto_error=False)
def hash_password(password: str) -> str:
return bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode()
def verify_password(password: str, hashed: str) -> bool:
try:
return bcrypt.checkpw(password.encode(), hashed.encode())
except Exception:
return False
def create_token(user_id: str, role: str, expires_minutes: int | None = None) -> str:
s = get_settings()
exp = datetime.now(timezone.utc) + timedelta(
minutes=expires_minutes if expires_minutes is not None else s.jwt_expire_minutes)
return jwt.encode({"sub": user_id, "role": role, "exp": exp}, s.jwt_secret, algorithm="HS256")
def decode_token(token: str) -> dict | None:
"""No DB hit: used by tracing middleware to attribute the session user."""
try:
return jwt.decode(token, get_settings().jwt_secret, algorithms=["HS256"])
except Exception:
return None
def _unauthorized(detail: str = "not authenticated") -> HTTPException:
return HTTPException(status.HTTP_401_UNAUTHORIZED, detail,
headers={"WWW-Authenticate": "Bearer"})
async def get_current_user(
creds: HTTPAuthorizationCredentials | None = Depends(_bearer),
db: AsyncSession = Depends(get_session),
):
from models import User # late import: avoids cycle at module load
if creds is None or not creds.credentials:
raise _unauthorized()
payload = decode_token(creds.credentials)
if not payload or not payload.get("sub"):
raise _unauthorized("invalid or expired token")
from sqlalchemy import select
user = (await db.execute(select(User).where(User.id == payload["sub"]))).scalar_one_or_none()
if user is None or user.status == "suspended":
raise _unauthorized("user not found or suspended")
return user

69
src/main.py Normal file
View file

@ -0,0 +1,69 @@
from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
from fastapi import Depends, FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from shared import events
from shared.logger import get_logger
from shared.tracing import TraceMiddleware
from shared.traffic import TrafficMiddleware
from src.api.v1 import activities, annotations, auth, events as events_api, images, projects, system
from src.api.v1.auth import users_router
from src.core.config import get_settings
from src.core.security import get_current_user
s = get_settings()
log = get_logger("danable")
@asynccontextmanager
async def lifespan(app: FastAPI):
drainer = asyncio.create_task(events.drain_forever())
yield
drainer.cancel()
app = FastAPI(title=s.app_name, lifespan=lifespan)
app.add_middleware(
CORSMiddleware,
allow_origins=["http://localhost:5173", "http://localhost:3000"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
if s.traffic.get("enabled", True):
app.add_middleware(TrafficMiddleware)
app.add_middleware(TraceMiddleware)
@app.exception_handler(RuntimeError)
async def _config_error(request: Request, exc: RuntimeError):
# Missing R2 creds etc: 503 with actionable message instead of 500.
return JSONResponse(status_code=503, detail=str(exc))
prefix = s.api_prefix
authed: list = [Depends(get_current_user)]
# Public: health/metrics, auth, client-event ingest (session telemetry).
app.include_router(system.router, prefix=prefix)
app.include_router(auth.router, prefix=prefix)
app.include_router(users_router, prefix=prefix, dependencies=authed)
app.include_router(events_api.router, prefix=prefix)
app.include_router(activities.router, prefix=prefix, dependencies=authed)
# Domain: require JWT.
app.include_router(projects.router, prefix=prefix, dependencies=authed)
app.include_router(images.router, prefix=prefix, dependencies=authed)
app.include_router(annotations.router, prefix=prefix, dependencies=authed)
@app.get("/")
async def root():
log.info("serving api prefix=%s", prefix)
return {"name": s.app_name, "docs": "/docs", "health": f"{prefix}/health"}

View file

@ -0,0 +1,58 @@
"""Annotation business logic: one row per image (upsert)."""
from __future__ import annotations
import time
from fastapi import HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from models import Annotation, Image
from shared import events
from shared.schemas import AnnotationUpsert
async def _require_image(project_id: str, image_id: str, db: AsyncSession) -> Image:
im = await db.get(Image, image_id)
if im is None or im.project_id != project_id:
raise HTTPException(404, "image not found")
return im
async def get_annotation(project_id: str, image_id: str, db: AsyncSession) -> dict:
ann = await db.get(Annotation, image_id)
if ann is None:
return {"imageId": image_id, "projectId": project_id,
"classificationLabelId": None,
"flags": {"blurryOrUncertain": False, "needsRecheck": False}, "shapes": []}
return {"imageId": ann.image_id, "projectId": ann.project_id,
"classificationLabelId": ann.classification_label_id,
"flags": ann.flags, "shapes": ann.shapes}
async def upsert_annotation(project_id: str, image_id: str, body: AnnotationUpsert,
db: AsyncSession) -> dict:
im = await _require_image(project_id, image_id, db)
t0 = time.perf_counter()
ann = await db.get(Annotation, image_id)
payload = {"project_id": project_id,
"classification_label_id": body.classificationLabelId,
"flags": body.flags.model_dump(),
"shapes": [s.model_dump() for s in body.shapes]}
if ann is None:
ann = Annotation(image_id=image_id, **payload)
db.add(ann)
else:
for k, v in payload.items():
setattr(ann, k, v)
im = await db.get(Image, image_id)
if im is not None:
im.object_count = len(body.shapes)
im.classification_label_id = body.classificationLabelId
im.flags = body.flags.model_dump()
await db.commit()
await events.emit("annotation.save", project_id=project_id, image_id=image_id,
payload={"shapeCount": len(body.shapes),
"label": body.classificationLabelId},
result={"saved": True},
latency_ms=(time.perf_counter() - t0) * 1000)
return {"imageId": image_id, "saved": True, "shapeCount": len(body.shapes)}

104
src/services/images.py Normal file
View file

@ -0,0 +1,104 @@
"""Image business logic: presign (direct-to-R2), confirm, status."""
from __future__ import annotations
import time
from fastapi import HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models import Image
from shared import events, keys, r2
from shared.schemas import ImageConfirm
def _thumb_url(*candidates: str | None) -> str:
for key in candidates:
if key:
return r2.presign_get(key)
return ""
async def list_images(project_id: str, status: str | None, db: AsyncSession,
search: str | None = None, assignee: str | None = None,
limit: int = 50, offset: int = 0) -> dict:
"""Paginated image list. One page = one screenful; never the whole dataset."""
from models import User
from sqlalchemy import func
limit = max(1, min(limit, 200))
offset = max(0, offset)
q = select(Image).where(Image.project_id == project_id)
count_q = select(func.count()).where(Image.project_id == project_id)
if status and status != "all":
q = q.where(Image.status == status)
count_q = count_q.where(Image.status == status)
if search and search.strip():
like = f"%{search.strip()}%"
q = q.where(Image.file_name.ilike(like))
count_q = count_q.where(Image.file_name.ilike(like))
if assignee and assignee != "all":
q = q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
count_q = count_q.join(User, User.id == Image.assigned_to).where(User.name == assignee)
total = (await db.execute(count_q)).scalar() or 0
rows = (await db.execute(q.order_by(Image.created_at.desc()).limit(limit).offset(offset))).scalars().all()
users = (await db.execute(select(User))).scalars().all()
names = {u.id: u.name for u in users}
items = [{"id": im.id, "projectId": im.project_id, "fileName": im.file_name,
"url": r2.presign_get(im.r2_key),
"thumbnailUrl": _thumb_url(im.thumbnail_r2_key, im.r2_key),
"thumbnailSmallUrl": _thumb_url(im.thumbnail_small_r2_key,
im.thumbnail_r2_key, im.r2_key),
"width": im.width, "height": im.height, "status": im.status,
"objectCount": im.object_count, "assignedTo": im.assigned_to,
"assignedToName": names.get(im.assigned_to or "", ""),
"reviewerName": im.reviewer_name,
"classificationLabelId": im.classification_label_id, "flags": im.flags}
for im in rows]
return {"items": items, "total": total}
def build_presign(project_id: str, file_name: str, content_type: str) -> dict:
"""Step 1/2 direct upload: frontend PUTs bytes straight to R2 with this URL."""
key = keys.image_key(project_id, "tmp", file_name) # real id assigned on confirm
return {"putUrl": r2.presign_put(key, content_type), "r2Key": key}
async def confirm_image(project_id: str, body: ImageConfirm, db: AsyncSession) -> dict:
"""Step 2/2: metadata row after the R2 PUT succeeds."""
if not r2.head_exists(body.r2Key):
raise HTTPException(400, "object not found in R2; upload first via presign PUT")
im = Image(project_id=project_id, file_name=body.fileName, r2_key=body.r2Key,
width=body.width, height=body.height, status="in_progress", flags={})
db.add(im)
t0 = time.perf_counter()
await db.commit()
await db.refresh(im)
try:
from shared import thumbnails
minted = await thumbnails.mint_thumbnails_for_image(project_id, im.id, im.r2_key)
if minted["thumbnail"]:
im.thumbnail_r2_key = minted["thumbnail"]
if minted["small"]:
im.thumbnail_small_r2_key = minted["small"]
if minted["thumbnail"] or minted["small"]:
await db.commit()
except Exception:
pass # card falls back to the full image
await events.emit("image.confirm", project_id=project_id, image_id=im.id,
payload={"fileName": body.fileName, "r2Key": body.r2Key},
latency_ms=(time.perf_counter() - t0) * 1000)
return {"id": im.id, "r2Key": im.r2_key, "url": r2.presign_get(im.r2_key)}
async def update_status(project_id: str, image_id: str, status: str, db: AsyncSession) -> dict:
im = await db.get(Image, image_id)
if im is None or im.project_id != project_id:
raise HTTPException(404, "image not found")
prev = im.status
im.status = status
t0 = time.perf_counter()
await db.commit()
await events.emit("image.status", project_id=project_id, image_id=im.id,
payload={"from": prev, "to": status},
latency_ms=(time.perf_counter() - t0) * 1000)
return {"id": im.id, "status": im.status}

184
src/services/projects.py Normal file
View file

@ -0,0 +1,184 @@
"""Project + label-class business logic (called by the API layer)."""
from __future__ import annotations
import time
from fastapi import HTTPException
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from models import AIJob, ActionEvent, Annotation, ExportJob, Image, LabelClass, Project, User
from shared import events
from shared.schemas import LabelClassIn, ProjectCreate
_MEMBER_COLORS = ["#E0E7FF", "#DBEAFE", "#FCE7F3", "#FFEDD5", "#EDE9FE", "#DCFCE7"]
def _initials(name: str) -> str:
parts = name.split()
if len(parts) >= 2:
return f"{parts[0][0]}{parts[-1][0]}".upper()
return (name[:2] or "??").upper()
async def list_projects(db: AsyncSession) -> list[dict]:
rows = (await db.execute(select(Project).order_by(Project.created_at.desc()))).scalars().all()
return [await _project_card(p, db) for p in rows]
async def create_project(body: ProjectCreate, db: AsyncSession) -> dict:
exists = (await db.execute(select(Project).where(Project.slug == body.slug))).scalar_one_or_none()
if exists:
raise HTTPException(409, "slug already exists")
p = Project(slug=body.slug, name=body.name, description=body.description,
task_type=body.taskType, status="annotating")
db.add(p)
t0 = time.perf_counter()
await db.commit()
await db.refresh(p)
await events.emit("project.create", project_id=p.id,
payload={"slug": p.slug, "taskType": p.task_type},
result={"id": p.id},
latency_ms=(time.perf_counter() - t0) * 1000)
return {"id": p.id, "slug": p.slug, "name": p.name, "taskType": p.task_type, "status": p.status}
async def _stats(project_id: str, db: AsyncSession) -> dict:
counts: dict[str, int] = {}
for st in ("unassigned", "in_progress", "pending_review", "completed"):
counts[st] = (await db.execute(
select(func.count()).where(Image.project_id == project_id, Image.status == st))).scalar() or 0
total = sum(counts.values())
objects = (await db.execute(
select(func.coalesce(func.sum(Image.object_count), 0)).where(Image.project_id == project_id))).scalar() or 0
return {"totalImages": total, "unassigned": counts["unassigned"], "inProgress": counts["in_progress"],
"pendingReview": counts["pending_review"], "completed": counts["completed"],
"totalObjects": int(objects)}
async def _members(project_id: str, db: AsyncSession) -> list[dict]:
users = (await db.execute(select(User).order_by(User.created_at))).scalars().all()
assigned = [u for u in users if project_id in (u.assigned_project_ids or [])]
out = []
for i, u in enumerate(assigned):
n = (await db.execute(select(func.count()).where(
Image.project_id == project_id, Image.assigned_to == u.id))).scalar() or 0
out.append({"id": u.id, "name": u.name, "initials": _initials(u.name),
"color": _MEMBER_COLORS[i % len(_MEMBER_COLORS)],
"role": u.role if u.role in ("annotator", "reviewer", "manager") else "annotator",
"assignedCount": n,
**({"pendingReviewCount": n} if u.role == "reviewer" else {})})
return out
async def _thumbnail_url(project_id: str, db: AsyncSession) -> str:
"""Project cover: dedicated thumb when present, else first image itself.
Trusts DB keys without R2 HEAD checks: a HEAD per card is 100-300ms of
API latency. Stale keys render nothing thanks to the UI onError fallback.
"""
from shared import r2
row = (await db.execute(
select(Image.thumbnail_r2_key, Image.r2_key)
.where(Image.project_id == project_id)
.order_by(Image.created_at).limit(1))).first()
if not row:
return ""
for key in (row[0], row[1]):
if not key:
continue
try:
return r2.presign_get(key)
except Exception:
continue
return ""
async def _project_card(p: Project, db: AsyncSession) -> dict:
classes = (await db.execute(select(LabelClass).where(LabelClass.project_id == p.id))).scalars().all()
return {"id": p.id, "slug": p.slug, "name": p.name, "taskType": p.task_type,
"status": p.status, "description": p.description,
"thumbnailUrl": await _thumbnail_url(p.id, db),
"updatedAt": p.updated_at.isoformat() if p.updated_at else "",
"classes": [{"id": c.id, "name": c.name, "color": c.color} for c in classes],
"stats": await _stats(p.id, db),
"members": await _members(p.id, db)}
async def get_project_detail(project_id: str, db: AsyncSession) -> dict:
p = await db.get(Project, project_id)
if p is None:
raise HTTPException(404, "project not found")
return await _project_card(p, db)
async def patch_project(project_id: str, patch: dict, db: AsyncSession) -> dict:
p = await db.get(Project, project_id)
if p is None:
raise HTTPException(404, "project not found")
for field in ("name", "slug", "description", "task_type", "status"):
if patch.get(field) is not None:
setattr(p, field, patch[field])
await db.commit()
return await get_project_detail(project_id, db)
async def delete_project(project_id: str, db: AsyncSession) -> dict:
p = await db.get(Project, project_id)
if p is None:
raise HTTPException(404, "project not found")
img_ids = (await db.execute(select(Image.id).where(Image.project_id == project_id))).scalars().all()
for img_id in img_ids:
ann = await db.get(Annotation, img_id)
if ann is not None:
await db.delete(ann)
im = await db.get(Image, img_id)
if im is not None:
await db.delete(im)
for model, col in ((ExportJob, ExportJob.project_id), (AIJob, AIJob.project_id),
(ActionEvent, ActionEvent.project_id), (LabelClass, LabelClass.project_id)):
rows = (await db.execute(select(model).where(col == project_id))).scalars().all()
for r in rows:
await db.delete(r)
await db.delete(p)
await db.commit()
return {"deleted": project_id}
async def replace_classes(project_id: str, classes: list[dict], db: AsyncSession) -> list[dict]:
"""Bulk replace: matches LabelsTab onUpdateClasses semantics (add/edit/delete in one call)."""
if await db.get(Project, project_id) is None:
raise HTTPException(404, "project not found")
existing = (await db.execute(select(LabelClass).where(LabelClass.project_id == project_id))).scalars().all()
by_id = {c.id: c for c in existing}
seen: set[str] = set()
out = []
for c in classes:
cid = c.get("id")
if cid in by_id:
obj = by_id[cid]
obj.name = c.get("name", obj.name)
obj.color = c.get("color", obj.color)
seen.add(cid)
else:
obj = LabelClass(project_id=project_id, name=c.get("name", "Untitled"),
color=c.get("color", "#888888"))
db.add(obj)
out.append(obj)
for cid, obj in by_id.items():
if cid not in seen:
await db.delete(obj)
await db.commit()
for obj in out:
await db.refresh(obj)
return [{"id": o.id, "name": o.name, "color": o.color} for o in out]
async def add_label_class(project_id: str, body: LabelClassIn, db: AsyncSession) -> dict:
if await db.get(Project, project_id) is None:
raise HTTPException(404, "project not found")
c = LabelClass(project_id=project_id, name=body.name, color=body.color, shortcut_key=body.shortcutKey)
db.add(c)
await db.commit()
await db.refresh(c)
return {"id": c.id, "name": c.name, "color": c.color}

60
src/services/users.py Normal file
View file

@ -0,0 +1,60 @@
"""User business logic: list / invite / patch role-status-assignment."""
from __future__ import annotations
import secrets
from fastapi import HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models import User
from src.core.security import hash_password
INVITE_PREFIX = "invite.local"
def to_public(u: User) -> dict:
return {"id": u.id, "email": u.email, "name": u.name, "role": u.role,
"status": u.status, "assignedProjectIds": u.assigned_project_ids or []}
async def list_users(db: AsyncSession) -> list[dict]:
rows = (await db.execute(select(User).order_by(User.created_at))).scalars().all()
return [to_public(u) for u in rows]
async def invite_user(email: str, role: str, project_ids: list[str], db: AsyncSession) -> dict:
email = email.strip().lower()
if not email or "@" not in email:
raise HTTPException(400, "invalid email")
exists = (await db.execute(select(User).where(User.email == email))).scalar_one_or_none()
if exists:
raise HTTPException(409, "email already invited")
name = " ".join(p.capitalize() for p in email.split("@")[0].replace(".", " ").replace("_", " ").split()) or email
u = User(email=email, name=name, role=role, status="invited",
# Unusable until password is set; login with it always fails.
password_hash=hash_password(secrets.token_urlsafe(32)),
assigned_project_ids=list(project_ids or []))
db.add(u)
await db.commit()
await db.refresh(u)
return to_public(u)
async def patch_user(user_id: str, patch: dict, db: AsyncSession) -> dict:
u = await db.get(User, user_id)
if u is None:
raise HTTPException(404, "user not found")
if patch.get("role") is not None:
if patch["role"] not in ("admin", "manager", "annotator", "reviewer"):
raise HTTPException(400, "invalid role")
u.role = patch["role"]
if patch.get("status") is not None:
if patch["status"] not in ("active", "invited", "suspended"):
raise HTTPException(400, "invalid status")
u.status = patch["status"]
if patch.get("projectIds") is not None:
u.assigned_project_ids = list(patch["projectIds"])
await db.commit()
await db.refresh(u)
return to_public(u)

8
stores/README.md Normal file
View file

@ -0,0 +1,8 @@
# Local dev mirror of the R2 bucket layout. Real bytes go to R2 in prod.
#
# stores/datasets/ <-> R2 datasets/{project}/... (uploaded images + thumbs/)
# stores/models/ <-> R2 models/{name}/{version}.pt (assist weights, never in git)
# stores/artifacts/ <-> R2 artifacts/... (exports, SAM embeddings cache)
#
# Source of truth for key shapes: shared/keys.py (mirrors config.yaml storage.layout).
# Drop sample files here for offline dev; the API presign flow writes straight to R2 instead.

View file

0
stores/datasets/.gitkeep Normal file
View file

0
stores/models/.gitkeep Normal file
View file

62
tests/test_annotations.py Normal file
View file

@ -0,0 +1,62 @@
"""Regression tests for the image/annotation flow (defects found in live smoke test)."""
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from src.core.db import Base
from src.services import annotations as svc
from shared.schemas import AnnotationUpsert, AnnotationShape
@pytest_asyncio.fixture
async def db():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
yield async_sessionmaker(engine, expire_on_commit=False)
await engine.dispose()
@pytest.mark.asyncio
async def test_raw_image_insert_gets_object_count_default(db):
# Reported defect: raw SQL insert without object_count violated NOT NULL.
async with db() as s:
await s.execute(text(
"INSERT INTO projects (id, slug, name, task_type, status) "
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
await s.execute(text(
"INSERT INTO images (id, project_id, file_name, r2_key, status, flags) "
"VALUES ('img1', 'p1', 'a.jpg', 'k', 'in_progress', '{}')"))
await s.commit()
n = (await s.execute(text("SELECT object_count FROM images WHERE id='img1'"))).scalar()
assert n == 0
@pytest.mark.asyncio
async def test_upsert_missing_image_is_404(db):
from fastapi import HTTPException
async with db() as s:
with pytest.raises(HTTPException) as e:
await svc.upsert_annotation("p1", "nope", AnnotationUpsert(), s)
assert e.value.status_code == 404
@pytest.mark.asyncio
async def test_upsert_roundtrip_syncs_object_count(db):
shape = AnnotationShape(id="s1", index=1, type="bbox", labelId="car",
bbox={"x": 1, "y": 2, "width": 10, "height": 10})
async with db() as s:
await s.execute(text(
"INSERT INTO projects (id, slug, name, task_type, status) "
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
await s.execute(text(
"INSERT INTO images (id, project_id, file_name, r2_key, status, flags) "
"VALUES ('img1', 'p1', 'a.jpg', 'k', 'in_progress', '{}')"))
await s.commit()
out = await svc.upsert_annotation("p1", "img1", AnnotationUpsert(shapes=[shape]), s)
assert out == {"imageId": "img1", "saved": True, "shapeCount": 1}
got = await svc.get_annotation("p1", "img1", s)
assert len(got["shapes"]) == 1
n = (await s.execute(text("SELECT object_count FROM images WHERE id='img1'"))).scalar()
assert n == 1

76
tests/test_auth.py Normal file
View file

@ -0,0 +1,76 @@
"""Auth: hashing, tokens, and route protection (no dev-DB writes)."""
import pytest
import pytest_asyncio
from fastapi.security import HTTPAuthorizationCredentials
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from models import User
from src.core.db import Base
from src.core.security import (
create_token,
decode_token,
get_current_user,
hash_password,
verify_password,
)
from src.main import app
from fastapi.testclient import TestClient
@pytest_asyncio.fixture
async def db():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
yield async_sessionmaker(engine, expire_on_commit=False)
await engine.dispose()
def test_password_roundtrip():
h = hash_password("Admin123!")
assert verify_password("Admin123!", h)
assert not verify_password("wrong", h)
assert not verify_password("x", "")
def test_token_roundtrip():
tok = create_token("u1", "admin")
payload = decode_token(tok)
assert payload and payload["sub"] == "u1" and payload["role"] == "admin"
assert decode_token("garbage") is None
@pytest.mark.asyncio
async def test_current_user_rejects_anonymous(db):
from fastapi import HTTPException
async with db() as s:
with pytest.raises(HTTPException) as e:
await get_current_user(None, s)
assert e.value.status_code == 401
@pytest.mark.asyncio
async def test_current_user_accepts_seeded(db):
async with db() as s:
s.add(User(id="u1", email="a@x.io", name="A", role="admin",
status="active", password_hash=hash_password("pw"),
assigned_project_ids=[]))
await s.commit()
tok = create_token("u1", "admin")
creds = HTTPAuthorizationCredentials(scheme="Bearer", credentials=tok)
user = await get_current_user(creds, s)
assert user.email == "a@x.io"
def test_protected_routes_reject_missing_token():
with TestClient(app) as c:
assert c.get("/api/v1/projects").status_code == 401
assert c.get("/api/v1/activities").status_code == 401
assert c.get("/api/v1/projects/x/images").status_code == 401
def test_login_rejects_unknown_user():
with TestClient(app) as c:
r = c.post("/api/v1/auth/login",
json={"email": "no-such-user@example.com", "password": "x"})
assert r.status_code == 401

62
tests/test_pagination.py Normal file
View file

@ -0,0 +1,62 @@
"""Pagination: one screenful per request, correct totals and filters."""
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from src.services import images as svc
@pytest_asyncio.fixture
async def db():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
from src.core.db import Base
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
yield async_sessionmaker(engine, expire_on_commit=False)
await engine.dispose()
async def _seed(db, n=5):
async with db() as s:
await s.execute(text(
"INSERT INTO projects (id, slug, name, task_type, status) "
"VALUES ('p1', 'p1', 'P', 'object_detection', 'annotating')"))
for i in range(n):
await s.execute(text(
"INSERT INTO images (id, project_id, file_name, r2_key, status, flags) "
f"VALUES ('img{i}', 'p1', 'f{i}.jpg', 'k{i}', "
f"'{'completed' if i % 2 == 0 else 'in_progress'}', '{{}}')"))
await s.commit()
@pytest.mark.asyncio
async def test_pages_cover_dataset_without_overlap(db):
await _seed(db, 5)
async with db() as s:
p1 = await svc.list_images("p1", "all", s, limit=2, offset=0)
p2 = await svc.list_images("p1", "all", s, limit=2, offset=2)
p3 = await svc.list_images("p1", "all", s, limit=2, offset=4)
assert p1["total"] == 5
ids = [i["id"] for i in p1["items"]] + [i["id"] for i in p2["items"]] \
+ [i["id"] for i in p3["items"]]
assert sorted(ids) == ["img0", "img1", "img2", "img3", "img4"]
assert len(p3["items"]) == 1
@pytest.mark.asyncio
async def test_status_and_search_filter_server_side(db):
await _seed(db, 5)
async with db() as s:
done = await svc.list_images("p1", "completed", s)
assert done["total"] == 3
one = await svc.list_images("p1", "all", s, search="f1")
assert one["total"] == 1 and one["items"][0]["id"] == "img1"
@pytest.mark.asyncio
async def test_limit_is_clamped(db):
await _seed(db, 2)
async with db() as s:
page = await svc.list_images("p1", "all", s, limit=10000, offset=-5)
assert page["total"] == 2 and len(page["items"]) == 2

30
tests/test_thumbnails.py Normal file
View file

@ -0,0 +1,30 @@
"""Thumbnail resize: pure bytes in/out, no R2, no DB."""
from io import BytesIO
from PIL import Image as PILImage
from shared.thumbnails import resize_to_thumbnail
def _jpeg(width: int, height: int) -> bytes:
buf = BytesIO()
PILImage.new("RGB", (width, height), (200, 30, 30)).save(buf, "JPEG", quality=90)
return buf.getvalue()
def test_resize_bounds_long_side():
out = resize_to_thumbnail(_jpeg(2000, 1000))
with PILImage.open(BytesIO(out)) as im:
assert max(im.size) <= 512
assert im.format == "JPEG"
def test_resize_small_image_not_upscaled():
out = resize_to_thumbnail(_jpeg(100, 60))
with PILImage.open(BytesIO(out)) as im:
assert im.size == (100, 60)
def test_resize_output_is_smaller():
big = _jpeg(1920, 1080)
assert len(resize_to_thumbnail(big)) < len(big)

42
tests/test_tracing.py Normal file
View file

@ -0,0 +1,42 @@
"""Trace layer: session echo, trace minting, client-event ingest."""
import uuid
import pytest
from fastapi.testclient import TestClient
from src.main import app
@pytest.fixture()
def client():
with TestClient(app) as c:
yield c
def test_trace_headers_echo_session(client):
r = client.get("/", headers={"X-Session-ID": "sess-1", "X-User-Id": "u1"})
assert r.status_code == 200
assert r.headers["X-Session-ID"] == "sess-1"
assert len(r.headers["X-Trace-ID"]) == 16
def test_trace_headers_mint_session_when_missing(client):
r = client.get("/")
assert r.status_code == 200
assert r.headers["X-Session-ID"]
assert r.headers["X-Trace-ID"]
def test_events_reject_missing_session(client):
r = client.post("/api/v1/events", json={"session_id": " ", "events": []})
assert r.status_code in (400, 422)
def test_events_accept_batch(client):
sid = f"sess-{uuid.uuid4().hex[:8]}"
r = client.post("/api/v1/events", json={
"session_id": sid,
"events": [{"action": "image.next", "project_id": "p1", "payload": {"dir": 1}}],
})
assert r.status_code == 202
assert r.json() == {"accepted": 1}