backend/shared/upload_worker.py

76 lines
2.6 KiB
Python

"""Redis-backed worker for turning uploaded objects into Image rows."""
from __future__ import annotations
import asyncio
import json
from sqlalchemy import select
from shared.logger import get_logger
from src.core.config import get_settings
log = get_logger("danagble.uploads")
QUEUE_KEY = "danagble:upload-items"
_redis = None
def _client():
global _redis
if _redis is None:
import redis.asyncio as aioredis
_redis = aioredis.from_url(get_settings().redis_url, decode_responses=True,
socket_connect_timeout=1, socket_timeout=10)
return _redis
async def enqueue(item_id: str) -> None:
await _client().lpush(QUEUE_KEY, json.dumps({"item_id": item_id}))
async def process(item_id: str) -> None:
from models import Image, UploadItem, UploadJob
from src.core.db import get_session_factory
factory = get_session_factory()
async with factory() as db:
item = await db.get(UploadItem, item_id)
if item is None or item.status != "uploaded":
return
job = await db.get(UploadJob, item.job_id)
if job is None:
return
image = Image(project_id=item.project_id, file_name=item.file_name,
r2_key=item.r2_key, width=item.width, height=item.height,
status="unassigned", flags={})
db.add(image)
item.status = "done"
item.image_id = image.id
job.processed_files += 1
if job.processed_files + job.failed_files >= job.total_files:
job.status = "done" if job.failed_files == 0 else "partial_error"
await db.commit()
await _client().publish(f"danagble:upload:{job.id}", json.dumps({
"jobId": job.id, "status": job.status,
"processedFiles": job.processed_files,
"totalFiles": job.total_files,
}))
async def run_forever() -> None:
log.info("upload worker started")
from models import UploadItem
from src.core.db import get_session_factory
factory = get_session_factory()
async with factory() as db:
rows = (await db.execute(select(UploadItem).where(
UploadItem.status == "uploaded"))).scalars().all()
for row in rows:
await enqueue(row.id)
while True:
try:
raw = await _client().brpop(QUEUE_KEY, timeout=5)
if raw:
await process(json.loads(raw[1])["item_id"])
except asyncio.CancelledError:
raise
except Exception as exc:
log.warning("upload worker error: %s", exc)
await asyncio.sleep(2)