From 5a546e4e4060d3042346bfcb60b1d235adca6ae6 Mon Sep 17 00:00:00 2001 From: hailt Date: Thu, 12 Nov 2026 22:44:29 +0700 Subject: [PATCH] feat: add Redis upload worker --- shared/upload_worker.py | 76 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 shared/upload_worker.py diff --git a/shared/upload_worker.py b/shared/upload_worker.py new file mode 100644 index 0000000..fc5b0eb --- /dev/null +++ b/shared/upload_worker.py @@ -0,0 +1,76 @@ +"""Redis-backed worker for turning uploaded objects into Image rows.""" +from __future__ import annotations + +import asyncio +import json + +from sqlalchemy import select +from shared.logger import get_logger +from src.core.config import get_settings + +log = get_logger("danagble.uploads") +QUEUE_KEY = "danagble:upload-items" +_redis = None + + +def _client(): + global _redis + if _redis is None: + import redis.asyncio as aioredis + _redis = aioredis.from_url(get_settings().redis_url, decode_responses=True, + socket_connect_timeout=1, socket_timeout=5) + return _redis + + +async def enqueue(item_id: str) -> None: + await _client().lpush(QUEUE_KEY, json.dumps({"item_id": item_id})) + + +async def process(item_id: str) -> None: + from models import Image, UploadItem, UploadJob + from src.core.db import get_session_factory + factory = get_session_factory() + async with factory() as db: + item = await db.get(UploadItem, item_id) + if item is None or item.status != "uploaded": + return + job = await db.get(UploadJob, item.job_id) + if job is None: + return + image = Image(project_id=item.project_id, file_name=item.file_name, + r2_key=item.r2_key, width=item.width, height=item.height, + status="unassigned", flags={}) + db.add(image) + item.status = "done" + item.image_id = image.id + job.processed_files += 1 + if job.processed_files + job.failed_files >= job.total_files: + job.status = "done" if job.failed_files == 0 else "partial_error" + await db.commit() + await _client().publish(f"danagble:upload:{job.id}", json.dumps({ + "jobId": job.id, "status": job.status, + "processedFiles": job.processed_files, + "totalFiles": job.total_files, + })) + + +async def run_forever() -> None: + log.info("upload worker started") + from models import UploadItem + from src.core.db import get_session_factory + factory = get_session_factory() + async with factory() as db: + rows = (await db.execute(select(UploadItem).where( + UploadItem.status == "uploaded"))).scalars().all() + for row in rows: + await enqueue(row.id) + while True: + try: + raw = await _client().brpop(QUEUE_KEY, timeout=5) + if raw: + await process(json.loads(raw[1])["item_id"]) + except asyncio.CancelledError: + raise + except Exception as exc: + log.warning("upload worker error: %s", exc) + await asyncio.sleep(2)