diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8adec81 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.env +.env.* +.git +.venv +.pytest_cache +__pycache__ +*.pyc diff --git a/.env.example b/.env.example index 24c68af..dac64d1 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,23 @@ # Copy to .env and fill secrets. NEVER commit .env. -# R2 = S3-compatible. Endpoint bucket-style: +# MinIO/R2 = S3-compatible. Endpoint bucket-style: # https://.r2.cloudflarestorage.com APP_ENV=dev API_HOST=0.0.0.0 API_PORT=8000 # --- Postgres / Redis (docker-compose provides defaults) --- -DATABASE_URL=postgresql+asyncpg://danable:danable@localhost:5432/danable -REDIS_URL=redis://localhost:6379/0 +POSTGRES_USER=danagble +POSTGRES_PASSWORD=change-this-password +POSTGRES_DB=danagble +DATABASE_URL=postgresql+asyncpg://danagble:change-this-password@postgres:5432/danagble +REDIS_URL=redis://redis:6379/0 # --- Auth --- JWT_SECRET=change-me JWT_EXPIRE_MINUTES=720 # --- Cloudflare R2 (secret part; non-secret shape lives in config.yaml) --- -R2_ENDPOINT=https://651cefb20bd66d84bd12cb62773b5a97.r2.cloudflarestorage.com +R2_ENDPOINT=http://minio:9000 R2_ACCESS_KEY_ID= R2_SECRET_ACCESS_KEY= R2_BUCKET=danagble diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..03a02b9 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,91 @@ +name: Deploy backend to prod (push-based) + +on: + workflow_run: + workflows: ["Backend CI"] + types: [completed] + branches: [main] + +concurrency: + group: prod-backend-deploy + cancel-in-progress: false + +jobs: + deploy: + name: Build image, push and roll out via bastion + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Checkout tested commit + uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha }} + + - name: Build backend image + run: docker build -t danagble-be:latest . + + - name: Setup SSH key and bastion config + env: + SSH_PRIVATE_KEY: ${{ secrets.PROD_SSH_KEY }} + run: | + set -euo pipefail + install -m 700 -d ~/.ssh + KEY="$(printf '%s' "$SSH_PRIVATE_KEY" | python3 -c ' + import re, sys + s = sys.stdin.read() + s = re.sub(r"\\+n", "\n", s) + s = s.replace("-----BEGIN OPENSSH PRIVATE KEY-----", "") + s = s.replace("-----END OPENSSH PRIVATE KEY-----", "") + body = "".join(s.split()) + sys.stdout.write("-----BEGIN OPENSSH PRIVATE KEY-----\n") + sys.stdout.write("\n".join(body[i:i + 64] for i in range(0, len(body), 64))) + sys.stdout.write("\n-----END OPENSSH PRIVATE KEY-----\n") + ')" + printf '%s\n' "$KEY" | tr -d '\r' > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keygen -lf ~/.ssh/id_ed25519 + cat > ~/.ssh/config <<'EOF' + Host bastion + HostName 103.195.236.32 + User root + IdentityFile ~/.ssh/id_ed25519 + StrictHostKeyChecking accept-new + Host prod + HostName 100.109.54.207 + User neurosus + IdentityFile ~/.ssh/id_ed25519 + ProxyJump bastion + StrictHostKeyChecking accept-new + EOF + chmod 600 ~/.ssh/config + + - name: Transfer compose and image + run: | + set -euo pipefail + tar -cf - docker-compose.prod.yml | ssh prod 'mkdir -p ~/danangble && tar -xf - -C ~/danangble' + docker save danagble-be:latest | ssh prod 'docker load' + + - name: Roll out backend + run: | + set -euo pipefail + ssh prod ' + set -euo pipefail + cd ~/danangble + test -f backend.env + docker compose -p danagble-backend -f docker-compose.prod.yml up -d + docker compose -p danagble-backend -f docker-compose.prod.yml ps + ' + + - name: Health check + run: | + ssh prod ' + set -euo pipefail + for i in $(seq 1 30); do + status=$(docker inspect -f "{{.State.Health.Status}}" danangble-be 2>/dev/null || true) + [ "$status" = healthy ] && exit 0 + [ "$status" = unhealthy ] && docker logs --tail 80 danangble-be && exit 1 + sleep 2 + done + docker logs --tail 80 danangble-be + exit 1 + ' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4240f50 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: Backend CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Test and build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements.txt + + - name: Install dependencies + run: python -m pip install -r requirements.txt + + - name: Test + run: python -m pytest tests/ -q + + - name: Build image + run: docker build -t danagble-be:ci . diff --git a/README.md b/README.md index 36c5cf2..3345cc6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# danable backend +# danagble backend -Backend API for danable, a computer-vision data labeling platform +Backend API for danagble, a computer-vision data labeling platform (classification, object detection, segmentation). Metadata lives in PostgreSQL; image bytes, datasets, models and exports live in Cloudflare R2 (S3-compatible). A Redis-backed action-event pipeline records every user @@ -30,7 +30,7 @@ curl -X POST localhost:8000/api/v1/admin/init-db `init-db` creates all tables and seeds the dev admin: -- email: `admin@danable.local` +- email: `admin@danagble.local` - password: `Admin123!` Change or remove this account before any shared deployment. API docs (Swagger): @@ -50,7 +50,7 @@ shape/thresholds from env. | `.env` key | Purpose | Default | |---|---|---| -| `DATABASE_URL` | Postgres DSN (asyncpg) | `postgresql+asyncpg://danable:danable@localhost:5432/danable` | +| `DATABASE_URL` | Postgres DSN (asyncpg) | `postgresql+asyncpg://danagble:danagble@localhost:5432/danagble` | | `REDIS_URL` | Redis for the event queue | `redis://localhost:6379/0` | | `JWT_SECRET` | Signs access/refresh tokens | `change-me` (must override) | | `JWT_EXPIRE_MINUTES` | Access token lifetime | `720` | diff --git a/config.example.yaml b/config.example.yaml index d31bd48..3417f9b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,7 +1,7 @@ # Non-secret backend configuration. Secrets ONLY in .env. # Rule: .env decides WHO/WHERE (credentials, hosts), yaml decides HOW (layout, ttl, thresholds). app: - name: danable + name: danagble api_prefix: /api/v1 api: diff --git a/config.yaml b/config.yaml index d31bd48..3417f9b 100644 --- a/config.yaml +++ b/config.yaml @@ -1,7 +1,7 @@ # Non-secret backend configuration. Secrets ONLY in .env. # Rule: .env decides WHO/WHERE (credentials, hosts), yaml decides HOW (layout, ttl, thresholds). app: - name: danable + name: danagble api_prefix: /api/v1 api: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..377b06f --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,70 @@ +name: danagble-backend + +services: + postgres: + image: postgres:16-alpine + container_name: danagble-postgres + restart: unless-stopped + env_file: + - ./backend.env + volumes: + - danagble_pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 10s + timeout: 5s + retries: 6 + networks: + - danagble + + redis: + image: redis:7-alpine + container_name: danagble-redis + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 6 + networks: + - danagble + + be: + image: danagble-be:latest + container_name: danagble-be + restart: unless-stopped + env_file: + - ./backend.env + expose: + - "8000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: uvicorn src.main:app --host 0.0.0.0 --port 8000 + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/v1/health', timeout=3) + interval: 10s + timeout: 5s + retries: 6 + start_period: 20s + networks: + - danagble + - neuraagent + +volumes: + danagble_pgdata: + +networks: + danagble: + external: true + # Existing FE network name; rename only together with the FE production compose. + name: danangble_default + neuraagent: + external: true + name: neuraagent_neura-network diff --git a/docker-compose.yml b/docker-compose.yml index de40da3..daa7911 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,22 +1,27 @@ +name: danagble + services: postgres: image: postgres:16-alpine + container_name: danagble-postgres environment: - POSTGRES_USER: danable - POSTGRES_PASSWORD: danable - POSTGRES_DB: danable + POSTGRES_USER: danagble + POSTGRES_PASSWORD: danagble + POSTGRES_DB: danagble ports: ["5432:5432"] volumes: [pgdata:/var/lib/postgresql/data] redis: image: redis:7-alpine + container_name: danagble-redis ports: ["6379:6379"] api: build: . + container_name: danagble-be env_file: .env environment: - DATABASE_URL: postgresql+asyncpg://danable:danable@postgres:5432/danable + DATABASE_URL: postgresql+asyncpg://danagble:danagble@postgres:5432/danagble REDIS_URL: redis://redis:6379/0 ports: ["8000:8000"] depends_on: [postgres, redis] diff --git a/docs/API.md b/docs/API.md index 8579f64..cb239d4 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,4 +1,4 @@ -# danable API reference +# danagble API reference Base URL: `http://localhost:8000/api/v1` (prefix from `config.yaml` `app.api_prefix`). Interactive docs: `http://localhost:8000/docs`. @@ -35,7 +35,7 @@ Interactive docs: `http://localhost:8000/docs`. ```json { - "user": { "id": "...", "name": "Admin", "email": "admin@danable.local", + "user": { "id": "...", "name": "Admin", "email": "admin@danagble.local", "avatarInitials": "A", "avatarColor": "#3B82F6", "role": "admin", "status": "active", "isOnline": true, "lastActive": "..." }, @@ -179,7 +179,7 @@ into `action_events` (requeued on DB failure, never lost, never blocks). B=http://localhost:8000/api/v1 curl -s -X POST $B/admin/init-db TOK=$(curl -s -X POST $B/auth/login -H 'Content-Type: application/json' \ - -d '{"email":"admin@danable.local","password":"Admin123!"}' \ + -d '{"email":"admin@danagble.local","password":"Admin123!"}' \ | python3 -c "import sys,json; print(json.load(sys.stdin)['accessToken'])") H="Authorization: Bearer $TOK" PID=$(curl -s -X POST $B/projects -H "$H" -H 'Content-Type: application/json' \ diff --git a/restart_api.sh b/restart_api.sh index 2e84c6e..3be61cc 100755 --- a/restart_api.sh +++ b/restart_api.sh @@ -1,9 +1,9 @@ #!/bin/bash -# Restart danable API safely: pattern lives in this file, never in the caller cmdline. +# Restart danagble API safely: pattern lives in this file, never in the caller cmdline. pkill -f 'uvicorn src\.main:app' 2>/dev/null sleep 2 cd /home/hailt/danagble/backend -setsid -f .venv/bin/python -m uvicorn src.main:app --host 0.0.0.0 --port 8000 >/tmp/danable-api.log 2>&1 +setsid -f .venv/bin/python -m uvicorn src.main:app --host 0.0.0.0 --port 8000 >/tmp/danagble-api.log 2>&1 sleep 4 curl -s -m 5 localhost:8000/api/v1/health echo diff --git a/shared/events.py b/shared/events.py index 38b6190..e0ce2aa 100644 --- a/shared/events.py +++ b/shared/events.py @@ -14,8 +14,8 @@ import time from shared.logger import get_logger, session_var, trace_var, user_var from src.core.config import get_settings -log = get_logger("danable.events") -QUEUE_KEY = "danable:events" +log = get_logger("danagble.events") +QUEUE_KEY = "danagble:events" _redis = None diff --git a/shared/logger.py b/shared/logger.py index fc393d2..1ce48e6 100644 --- a/shared/logger.py +++ b/shared/logger.py @@ -12,9 +12,9 @@ import logging import sys from datetime import datetime, timezone -session_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.session_id", default="") -trace_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.trace_id", default="") -user_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.user_id", default="anonymous") +session_var: contextvars.ContextVar[str] = contextvars.ContextVar("danagble.session_id", default="") +trace_var: contextvars.ContextVar[str] = contextvars.ContextVar("danagble.trace_id", default="") +user_var: contextvars.ContextVar[str] = contextvars.ContextVar("danagble.user_id", default="anonymous") _configured = False @@ -35,12 +35,12 @@ class JsonFormatter(logging.Formatter): return json.dumps(payload, ensure_ascii=False) -def get_logger(name: str = "danable") -> logging.Logger: +def get_logger(name: str = "danagble") -> logging.Logger: global _configured if not _configured: handler = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) - root = logging.getLogger("danable") + root = logging.getLogger("danagble") root.handlers.clear() root.addHandler(handler) root.setLevel(logging.INFO) diff --git a/shared/thumbnails.py b/shared/thumbnails.py index a20d548..fb0aa6a 100644 --- a/shared/thumbnails.py +++ b/shared/thumbnails.py @@ -12,7 +12,7 @@ from shared import keys, r2 from shared.logger import get_logger from src.core.config import get_settings -log = get_logger("danable.thumbnails") +log = get_logger("danagble.thumbnails") def resize_to_thumbnail(data: bytes, max_side: int = 512, quality: int = 70) -> bytes: diff --git a/shared/tracing.py b/shared/tracing.py index 525d0a0..c24b9e2 100644 --- a/shared/tracing.py +++ b/shared/tracing.py @@ -13,7 +13,7 @@ from starlette.requests import Request from shared.logger import get_logger, session_var, trace_var, user_var -log = get_logger("danable.trace") +log = get_logger("danagble.trace") SESSION_HEADER = "X-Session-ID" TRACE_HEADER = "X-Trace-ID" diff --git a/shared/traffic.py b/shared/traffic.py index 4fafd4f..f5b1c18 100644 --- a/shared/traffic.py +++ b/shared/traffic.py @@ -14,7 +14,7 @@ from starlette.requests import Request from shared.logger import get_logger from src.core.config import get_settings -log = get_logger("danable.traffic") +log = get_logger("danagble.traffic") _counters: dict[str, dict[str, float]] = defaultdict(lambda: {"count": 0, "errors": 0, "total_ms": 0.0}) _started_at = time.time() diff --git a/src/api/v1/system.py b/src/api/v1/system.py index 24d248a..7e9a1a4 100644 --- a/src/api/v1/system.py +++ b/src/api/v1/system.py @@ -73,10 +73,10 @@ async def init_db(): await conn.run_sync(Base.metadata.create_all) factory = get_session_factory() async with factory() as s: - admin = (await s.execute(select(User).where(User.email == "admin@danable.local"))).scalar_one_or_none() + admin = (await s.execute(select(User).where(User.email == "admin@danagble.local"))).scalar_one_or_none() if admin is None: - s.add(User(email="admin@danable.local", name="Admin", role="admin", + s.add(User(email="admin@danagble.local", name="Admin", role="admin", status="active", password_hash=hash_password("Admin123!"), assigned_project_ids=[])) await s.commit() - return {"ok": True, "seeded_admin": "admin@danable.local"} + return {"ok": True, "seeded_admin": "admin@danagble.local"} diff --git a/src/core/config.py b/src/core/config.py index 7c45802..0db7059 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -25,11 +25,11 @@ class Settings: self.yaml = _load_yaml() app = self.yaml.get("app", {}) storage = self.yaml.get("storage", {}) - self.app_name: str = app.get("name", "danable") + self.app_name: str = app.get("name", "danagble") self.api_prefix: str = app.get("api_prefix", "/api/v1") self.database_url: str = os.getenv( - "DATABASE_URL", "postgresql+asyncpg://danable:danable@localhost:5432/danable" + "DATABASE_URL", "postgresql+asyncpg://danagble:danagble@localhost:5432/danagble" ) self.redis_url: str = os.getenv("REDIS_URL", "redis://localhost:6379/0") self.jwt_secret: str = os.getenv("JWT_SECRET", "change-me") diff --git a/src/main.py b/src/main.py index 13050a4..3e9ff4b 100644 --- a/src/main.py +++ b/src/main.py @@ -17,7 +17,7 @@ from src.core.config import get_settings from src.core.security import get_current_user s = get_settings() -log = get_logger("danable") +log = get_logger("danagble") @asynccontextmanager diff --git a/src/services/imports.py b/src/services/imports.py index 00d2d82..d409044 100644 --- a/src/services/imports.py +++ b/src/services/imports.py @@ -24,7 +24,7 @@ from shared.schemas import AnnotationShape, AnnotationUpsert, Flags from shared.thumbnails import IMMUTABLE, resize_to_thumbnail from src.core.db import get_session_factory -log = get_logger("danable.imports") +log = get_logger("danagble.imports") IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp", ".bmp") PUT_CONCURRENCY = 24 @@ -135,7 +135,7 @@ async def start_r2_zip(project_id: str, r2_key: str, db: AsyncSession) -> dict: async def _run_r2_zip(job_id: str, project_id: str, r2_key: str) -> None: import tempfile - tmp = tempfile.mktemp(prefix=f"danable-import-{job_id}-", suffix=".zip") + tmp = tempfile.mktemp(prefix=f"danagble-import-{job_id}-", suffix=".zip") try: await asyncio.to_thread(r2.download_to_file, r2_key, tmp) await _run_zip_coco(job_id, project_id, tmp) diff --git a/tests/test_auth.py b/tests/test_auth.py index 72e0aea..c1d697e 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,11 +1,13 @@ """Auth: hashing, tokens, and route protection (no dev-DB writes).""" import pytest import pytest_asyncio +from httpx import ASGITransport, AsyncClient from fastapi.security import HTTPAuthorizationCredentials from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from models import User from src.core.db import Base +from src.core.db import get_session from src.core.security import ( create_token, decode_token, @@ -69,8 +71,23 @@ def test_protected_routes_reject_missing_token(): assert c.get("/api/v1/projects/x/images").status_code == 401 -def test_login_rejects_unknown_user(): - with TestClient(app) as c: - r = c.post("/api/v1/auth/login", - json={"email": "no-such-user@example.com", "password": "x"}) +@pytest.mark.asyncio +async def test_login_rejects_unknown_user(): + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + + async def override_get_session(): + async with session_factory() as session: + yield session + + app.dependency_overrides[get_session] = override_get_session + try: + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + r = await c.post("/api/v1/auth/login", + json={"email": "no-such-user@example.com", "password": "x"}) assert r.status_code == 401 + finally: + app.dependency_overrides.clear() + await engine.dispose()