"""Redis-backed worker for turning uploaded objects into Image rows.""" from __future__ import annotations import asyncio import json from sqlalchemy import select from shared.logger import get_logger from src.core.config import get_settings log = get_logger("danagble.uploads") QUEUE_KEY = "danagble:upload-items" _redis = None def _client(): global _redis if _redis is None: import redis.asyncio as aioredis _redis = aioredis.from_url(get_settings().redis_url, decode_responses=True, socket_connect_timeout=1, socket_timeout=5) return _redis async def enqueue(item_id: str) -> None: await _client().lpush(QUEUE_KEY, json.dumps({"item_id": item_id})) async def process(item_id: str) -> None: from models import Image, UploadItem, UploadJob from src.core.db import get_session_factory factory = get_session_factory() async with factory() as db: item = await db.get(UploadItem, item_id) if item is None or item.status != "uploaded": return job = await db.get(UploadJob, item.job_id) if job is None: return image = Image(project_id=item.project_id, file_name=item.file_name, r2_key=item.r2_key, width=item.width, height=item.height, status="unassigned", flags={}) db.add(image) item.status = "done" item.image_id = image.id job.processed_files += 1 if job.processed_files + job.failed_files >= job.total_files: job.status = "done" if job.failed_files == 0 else "partial_error" await db.commit() await _client().publish(f"danagble:upload:{job.id}", json.dumps({ "jobId": job.id, "status": job.status, "processedFiles": job.processed_files, "totalFiles": job.total_files, })) async def run_forever() -> None: log.info("upload worker started") from models import UploadItem from src.core.db import get_session_factory factory = get_session_factory() async with factory() as db: rows = (await db.execute(select(UploadItem).where( UploadItem.status == "uploaded"))).scalars().all() for row in rows: await enqueue(row.id) while True: try: raw = await _client().brpop(QUEUE_KEY, timeout=5) if raw: await process(json.loads(raw[1])["item_id"]) except asyncio.CancelledError: raise except Exception as exc: log.warning("upload worker error: %s", exc) await asyncio.sleep(2)