feat: add backend CI/CD deployment

This commit is contained in:
hailt 2026-11-12 21:32:31 +07:00
parent 8d40cff227
commit 7c2e6ed571
21 changed files with 269 additions and 41 deletions

7
.dockerignore Normal file
View file

@ -0,0 +1,7 @@
.env
.env.*
.git
.venv
.pytest_cache
__pycache__
*.pyc

View file

@ -1,20 +1,23 @@
# Copy to .env and fill secrets. NEVER commit .env.
# R2 = S3-compatible. Endpoint bucket-style:
# MinIO/R2 = S3-compatible. Endpoint bucket-style:
# https://<account_id>.r2.cloudflarestorage.com
APP_ENV=dev
API_HOST=0.0.0.0
API_PORT=8000
# --- Postgres / Redis (docker-compose provides defaults) ---
DATABASE_URL=postgresql+asyncpg://danable:danable@localhost:5432/danable
REDIS_URL=redis://localhost:6379/0
POSTGRES_USER=danagble
POSTGRES_PASSWORD=change-this-password
POSTGRES_DB=danagble
DATABASE_URL=postgresql+asyncpg://danagble:change-this-password@postgres:5432/danagble
REDIS_URL=redis://redis:6379/0
# --- Auth ---
JWT_SECRET=change-me
JWT_EXPIRE_MINUTES=720
# --- Cloudflare R2 (secret part; non-secret shape lives in config.yaml) ---
R2_ENDPOINT=https://651cefb20bd66d84bd12cb62773b5a97.r2.cloudflarestorage.com
R2_ENDPOINT=http://minio:9000
R2_ACCESS_KEY_ID=
R2_SECRET_ACCESS_KEY=
R2_BUCKET=danagble

91
.github/workflows/cd.yml vendored Normal file
View file

@ -0,0 +1,91 @@
name: Deploy backend to prod (push-based)
on:
workflow_run:
workflows: ["Backend CI"]
types: [completed]
branches: [main]
concurrency:
group: prod-backend-deploy
cancel-in-progress: false
jobs:
deploy:
name: Build image, push and roll out via bastion
if: ${{ github.event.workflow_run.conclusion == 'success' }}
runs-on: ubuntu-latest
steps:
- name: Checkout tested commit
uses: actions/checkout@v4
with:
ref: ${{ github.event.workflow_run.head_sha }}
- name: Build backend image
run: docker build -t danagble-be:latest .
- name: Setup SSH key and bastion config
env:
SSH_PRIVATE_KEY: ${{ secrets.PROD_SSH_KEY }}
run: |
set -euo pipefail
install -m 700 -d ~/.ssh
KEY="$(printf '%s' "$SSH_PRIVATE_KEY" | python3 -c '
import re, sys
s = sys.stdin.read()
s = re.sub(r"\\+n", "\n", s)
s = s.replace("-----BEGIN OPENSSH PRIVATE KEY-----", "")
s = s.replace("-----END OPENSSH PRIVATE KEY-----", "")
body = "".join(s.split())
sys.stdout.write("-----BEGIN OPENSSH PRIVATE KEY-----\n")
sys.stdout.write("\n".join(body[i:i + 64] for i in range(0, len(body), 64)))
sys.stdout.write("\n-----END OPENSSH PRIVATE KEY-----\n")
')"
printf '%s\n' "$KEY" | tr -d '\r' > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keygen -lf ~/.ssh/id_ed25519
cat > ~/.ssh/config <<'EOF'
Host bastion
HostName 103.195.236.32
User root
IdentityFile ~/.ssh/id_ed25519
StrictHostKeyChecking accept-new
Host prod
HostName 100.109.54.207
User neurosus
IdentityFile ~/.ssh/id_ed25519
ProxyJump bastion
StrictHostKeyChecking accept-new
EOF
chmod 600 ~/.ssh/config
- name: Transfer compose and image
run: |
set -euo pipefail
tar -cf - docker-compose.prod.yml | ssh prod 'mkdir -p ~/danangble && tar -xf - -C ~/danangble'
docker save danagble-be:latest | ssh prod 'docker load'
- name: Roll out backend
run: |
set -euo pipefail
ssh prod '
set -euo pipefail
cd ~/danangble
test -f backend.env
docker compose -p danagble-backend -f docker-compose.prod.yml up -d
docker compose -p danagble-backend -f docker-compose.prod.yml ps
'
- name: Health check
run: |
ssh prod '
set -euo pipefail
for i in $(seq 1 30); do
status=$(docker inspect -f "{{.State.Health.Status}}" danangble-be 2>/dev/null || true)
[ "$status" = healthy ] && exit 0
[ "$status" = unhealthy ] && docker logs --tail 80 danangble-be && exit 1
sleep 2
done
docker logs --tail 80 danangble-be
exit 1
'

35
.github/workflows/ci.yml vendored Normal file
View file

@ -0,0 +1,35 @@
name: Backend CI
on:
push:
branches: [main]
pull_request:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
name: Test and build
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: requirements.txt
- name: Install dependencies
run: python -m pip install -r requirements.txt
- name: Test
run: python -m pytest tests/ -q
- name: Build image
run: docker build -t danagble-be:ci .

View file

@ -1,6 +1,6 @@
# danable backend
# danagble backend
Backend API for danable, a computer-vision data labeling platform
Backend API for danagble, a computer-vision data labeling platform
(classification, object detection, segmentation). Metadata lives in
PostgreSQL; image bytes, datasets, models and exports live in Cloudflare R2
(S3-compatible). A Redis-backed action-event pipeline records every user
@ -30,7 +30,7 @@ curl -X POST localhost:8000/api/v1/admin/init-db
`init-db` creates all tables and seeds the dev admin:
- email: `admin@danable.local`
- email: `admin@danagble.local`
- password: `Admin123!`
Change or remove this account before any shared deployment. API docs (Swagger):
@ -50,7 +50,7 @@ shape/thresholds from env.
| `.env` key | Purpose | Default |
|---|---|---|
| `DATABASE_URL` | Postgres DSN (asyncpg) | `postgresql+asyncpg://danable:danable@localhost:5432/danable` |
| `DATABASE_URL` | Postgres DSN (asyncpg) | `postgresql+asyncpg://danagble:danagble@localhost:5432/danagble` |
| `REDIS_URL` | Redis for the event queue | `redis://localhost:6379/0` |
| `JWT_SECRET` | Signs access/refresh tokens | `change-me` (must override) |
| `JWT_EXPIRE_MINUTES` | Access token lifetime | `720` |

View file

@ -1,7 +1,7 @@
# Non-secret backend configuration. Secrets ONLY in .env.
# Rule: .env decides WHO/WHERE (credentials, hosts), yaml decides HOW (layout, ttl, thresholds).
app:
name: danable
name: danagble
api_prefix: /api/v1
api:

View file

@ -1,7 +1,7 @@
# Non-secret backend configuration. Secrets ONLY in .env.
# Rule: .env decides WHO/WHERE (credentials, hosts), yaml decides HOW (layout, ttl, thresholds).
app:
name: danable
name: danagble
api_prefix: /api/v1
api:

70
docker-compose.prod.yml Normal file
View file

@ -0,0 +1,70 @@
name: danagble-backend
services:
postgres:
image: postgres:16-alpine
container_name: danagble-postgres
restart: unless-stopped
env_file:
- ./backend.env
volumes:
- danagble_pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 10s
timeout: 5s
retries: 6
networks:
- danagble
redis:
image: redis:7-alpine
container_name: danagble-redis
restart: unless-stopped
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 6
networks:
- danagble
be:
image: danagble-be:latest
container_name: danagble-be
restart: unless-stopped
env_file:
- ./backend.env
expose:
- "8000"
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
command: uvicorn src.main:app --host 0.0.0.0 --port 8000
healthcheck:
test:
- CMD
- python
- -c
- import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/v1/health', timeout=3)
interval: 10s
timeout: 5s
retries: 6
start_period: 20s
networks:
- danagble
- neuraagent
volumes:
danagble_pgdata:
networks:
danagble:
external: true
# Existing FE network name; rename only together with the FE production compose.
name: danangble_default
neuraagent:
external: true
name: neuraagent_neura-network

View file

@ -1,22 +1,27 @@
name: danagble
services:
postgres:
image: postgres:16-alpine
container_name: danagble-postgres
environment:
POSTGRES_USER: danable
POSTGRES_PASSWORD: danable
POSTGRES_DB: danable
POSTGRES_USER: danagble
POSTGRES_PASSWORD: danagble
POSTGRES_DB: danagble
ports: ["5432:5432"]
volumes: [pgdata:/var/lib/postgresql/data]
redis:
image: redis:7-alpine
container_name: danagble-redis
ports: ["6379:6379"]
api:
build: .
container_name: danagble-be
env_file: .env
environment:
DATABASE_URL: postgresql+asyncpg://danable:danable@postgres:5432/danable
DATABASE_URL: postgresql+asyncpg://danagble:danagble@postgres:5432/danagble
REDIS_URL: redis://redis:6379/0
ports: ["8000:8000"]
depends_on: [postgres, redis]

View file

@ -1,4 +1,4 @@
# danable API reference
# danagble API reference
Base URL: `http://localhost:8000/api/v1` (prefix from `config.yaml` `app.api_prefix`).
Interactive docs: `http://localhost:8000/docs`.
@ -35,7 +35,7 @@ Interactive docs: `http://localhost:8000/docs`.
```json
{
"user": { "id": "...", "name": "Admin", "email": "admin@danable.local",
"user": { "id": "...", "name": "Admin", "email": "admin@danagble.local",
"avatarInitials": "A", "avatarColor": "#3B82F6",
"role": "admin", "status": "active",
"isOnline": true, "lastActive": "..." },
@ -179,7 +179,7 @@ into `action_events` (requeued on DB failure, never lost, never blocks).
B=http://localhost:8000/api/v1
curl -s -X POST $B/admin/init-db
TOK=$(curl -s -X POST $B/auth/login -H 'Content-Type: application/json' \
-d '{"email":"admin@danable.local","password":"Admin123!"}' \
-d '{"email":"admin@danagble.local","password":"Admin123!"}' \
| python3 -c "import sys,json; print(json.load(sys.stdin)['accessToken'])")
H="Authorization: Bearer $TOK"
PID=$(curl -s -X POST $B/projects -H "$H" -H 'Content-Type: application/json' \

View file

@ -1,9 +1,9 @@
#!/bin/bash
# Restart danable API safely: pattern lives in this file, never in the caller cmdline.
# Restart danagble API safely: pattern lives in this file, never in the caller cmdline.
pkill -f 'uvicorn src\.main:app' 2>/dev/null
sleep 2
cd /home/hailt/danagble/backend
setsid -f .venv/bin/python -m uvicorn src.main:app --host 0.0.0.0 --port 8000 </dev/null >>/tmp/danable-api.log 2>&1
setsid -f .venv/bin/python -m uvicorn src.main:app --host 0.0.0.0 --port 8000 </dev/null >>/tmp/danagble-api.log 2>&1
sleep 4
curl -s -m 5 localhost:8000/api/v1/health
echo

View file

@ -14,8 +14,8 @@ import time
from shared.logger import get_logger, session_var, trace_var, user_var
from src.core.config import get_settings
log = get_logger("danable.events")
QUEUE_KEY = "danable:events"
log = get_logger("danagble.events")
QUEUE_KEY = "danagble:events"
_redis = None

View file

@ -12,9 +12,9 @@ import logging
import sys
from datetime import datetime, timezone
session_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.session_id", default="")
trace_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.trace_id", default="")
user_var: contextvars.ContextVar[str] = contextvars.ContextVar("danable.user_id", default="anonymous")
session_var: contextvars.ContextVar[str] = contextvars.ContextVar("danagble.session_id", default="")
trace_var: contextvars.ContextVar[str] = contextvars.ContextVar("danagble.trace_id", default="")
user_var: contextvars.ContextVar[str] = contextvars.ContextVar("danagble.user_id", default="anonymous")
_configured = False
@ -35,12 +35,12 @@ class JsonFormatter(logging.Formatter):
return json.dumps(payload, ensure_ascii=False)
def get_logger(name: str = "danable") -> logging.Logger:
def get_logger(name: str = "danagble") -> logging.Logger:
global _configured
if not _configured:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger("danable")
root = logging.getLogger("danagble")
root.handlers.clear()
root.addHandler(handler)
root.setLevel(logging.INFO)

View file

@ -12,7 +12,7 @@ from shared import keys, r2
from shared.logger import get_logger
from src.core.config import get_settings
log = get_logger("danable.thumbnails")
log = get_logger("danagble.thumbnails")
def resize_to_thumbnail(data: bytes, max_side: int = 512, quality: int = 70) -> bytes:

View file

@ -13,7 +13,7 @@ from starlette.requests import Request
from shared.logger import get_logger, session_var, trace_var, user_var
log = get_logger("danable.trace")
log = get_logger("danagble.trace")
SESSION_HEADER = "X-Session-ID"
TRACE_HEADER = "X-Trace-ID"

View file

@ -14,7 +14,7 @@ from starlette.requests import Request
from shared.logger import get_logger
from src.core.config import get_settings
log = get_logger("danable.traffic")
log = get_logger("danagble.traffic")
_counters: dict[str, dict[str, float]] = defaultdict(lambda: {"count": 0, "errors": 0, "total_ms": 0.0})
_started_at = time.time()

View file

@ -73,10 +73,10 @@ async def init_db():
await conn.run_sync(Base.metadata.create_all)
factory = get_session_factory()
async with factory() as s:
admin = (await s.execute(select(User).where(User.email == "admin@danable.local"))).scalar_one_or_none()
admin = (await s.execute(select(User).where(User.email == "admin@danagble.local"))).scalar_one_or_none()
if admin is None:
s.add(User(email="admin@danable.local", name="Admin", role="admin",
s.add(User(email="admin@danagble.local", name="Admin", role="admin",
status="active", password_hash=hash_password("Admin123!"),
assigned_project_ids=[]))
await s.commit()
return {"ok": True, "seeded_admin": "admin@danable.local"}
return {"ok": True, "seeded_admin": "admin@danagble.local"}

View file

@ -25,11 +25,11 @@ class Settings:
self.yaml = _load_yaml()
app = self.yaml.get("app", {})
storage = self.yaml.get("storage", {})
self.app_name: str = app.get("name", "danable")
self.app_name: str = app.get("name", "danagble")
self.api_prefix: str = app.get("api_prefix", "/api/v1")
self.database_url: str = os.getenv(
"DATABASE_URL", "postgresql+asyncpg://danable:danable@localhost:5432/danable"
"DATABASE_URL", "postgresql+asyncpg://danagble:danagble@localhost:5432/danagble"
)
self.redis_url: str = os.getenv("REDIS_URL", "redis://localhost:6379/0")
self.jwt_secret: str = os.getenv("JWT_SECRET", "change-me")

View file

@ -17,7 +17,7 @@ from src.core.config import get_settings
from src.core.security import get_current_user
s = get_settings()
log = get_logger("danable")
log = get_logger("danagble")
@asynccontextmanager

View file

@ -24,7 +24,7 @@ from shared.schemas import AnnotationShape, AnnotationUpsert, Flags
from shared.thumbnails import IMMUTABLE, resize_to_thumbnail
from src.core.db import get_session_factory
log = get_logger("danable.imports")
log = get_logger("danagble.imports")
IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp", ".bmp")
PUT_CONCURRENCY = 24
@ -135,7 +135,7 @@ async def start_r2_zip(project_id: str, r2_key: str, db: AsyncSession) -> dict:
async def _run_r2_zip(job_id: str, project_id: str, r2_key: str) -> None:
import tempfile
tmp = tempfile.mktemp(prefix=f"danable-import-{job_id}-", suffix=".zip")
tmp = tempfile.mktemp(prefix=f"danagble-import-{job_id}-", suffix=".zip")
try:
await asyncio.to_thread(r2.download_to_file, r2_key, tmp)
await _run_zip_coco(job_id, project_id, tmp)

View file

@ -1,11 +1,13 @@
"""Auth: hashing, tokens, and route protection (no dev-DB writes)."""
import pytest
import pytest_asyncio
from httpx import ASGITransport, AsyncClient
from fastapi.security import HTTPAuthorizationCredentials
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from models import User
from src.core.db import Base
from src.core.db import get_session
from src.core.security import (
create_token,
decode_token,
@ -69,8 +71,23 @@ def test_protected_routes_reject_missing_token():
assert c.get("/api/v1/projects/x/images").status_code == 401
def test_login_rejects_unknown_user():
with TestClient(app) as c:
r = c.post("/api/v1/auth/login",
@pytest.mark.asyncio
async def test_login_rejects_unknown_user():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
session_factory = async_sessionmaker(engine, expire_on_commit=False)
async def override_get_session():
async with session_factory() as session:
yield session
app.dependency_overrides[get_session] = override_get_session
try:
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
r = await c.post("/api/v1/auth/login",
json={"email": "no-such-user@example.com", "password": "x"})
assert r.status_code == 401
finally:
app.dependency_overrides.clear()
await engine.dispose()