feat: add Redis upload worker
This commit is contained in:
parent
cee69acbdc
commit
5a546e4e40
1 changed files with 76 additions and 0 deletions
76
shared/upload_worker.py
Normal file
76
shared/upload_worker.py
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
"""Redis-backed worker for turning uploaded objects into Image rows."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from shared.logger import get_logger
|
||||||
|
from src.core.config import get_settings
|
||||||
|
|
||||||
|
log = get_logger("danagble.uploads")
|
||||||
|
QUEUE_KEY = "danagble:upload-items"
|
||||||
|
_redis = None
|
||||||
|
|
||||||
|
|
||||||
|
def _client():
|
||||||
|
global _redis
|
||||||
|
if _redis is None:
|
||||||
|
import redis.asyncio as aioredis
|
||||||
|
_redis = aioredis.from_url(get_settings().redis_url, decode_responses=True,
|
||||||
|
socket_connect_timeout=1, socket_timeout=5)
|
||||||
|
return _redis
|
||||||
|
|
||||||
|
|
||||||
|
async def enqueue(item_id: str) -> None:
|
||||||
|
await _client().lpush(QUEUE_KEY, json.dumps({"item_id": item_id}))
|
||||||
|
|
||||||
|
|
||||||
|
async def process(item_id: str) -> None:
|
||||||
|
from models import Image, UploadItem, UploadJob
|
||||||
|
from src.core.db import get_session_factory
|
||||||
|
factory = get_session_factory()
|
||||||
|
async with factory() as db:
|
||||||
|
item = await db.get(UploadItem, item_id)
|
||||||
|
if item is None or item.status != "uploaded":
|
||||||
|
return
|
||||||
|
job = await db.get(UploadJob, item.job_id)
|
||||||
|
if job is None:
|
||||||
|
return
|
||||||
|
image = Image(project_id=item.project_id, file_name=item.file_name,
|
||||||
|
r2_key=item.r2_key, width=item.width, height=item.height,
|
||||||
|
status="unassigned", flags={})
|
||||||
|
db.add(image)
|
||||||
|
item.status = "done"
|
||||||
|
item.image_id = image.id
|
||||||
|
job.processed_files += 1
|
||||||
|
if job.processed_files + job.failed_files >= job.total_files:
|
||||||
|
job.status = "done" if job.failed_files == 0 else "partial_error"
|
||||||
|
await db.commit()
|
||||||
|
await _client().publish(f"danagble:upload:{job.id}", json.dumps({
|
||||||
|
"jobId": job.id, "status": job.status,
|
||||||
|
"processedFiles": job.processed_files,
|
||||||
|
"totalFiles": job.total_files,
|
||||||
|
}))
|
||||||
|
|
||||||
|
|
||||||
|
async def run_forever() -> None:
|
||||||
|
log.info("upload worker started")
|
||||||
|
from models import UploadItem
|
||||||
|
from src.core.db import get_session_factory
|
||||||
|
factory = get_session_factory()
|
||||||
|
async with factory() as db:
|
||||||
|
rows = (await db.execute(select(UploadItem).where(
|
||||||
|
UploadItem.status == "uploaded"))).scalars().all()
|
||||||
|
for row in rows:
|
||||||
|
await enqueue(row.id)
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
raw = await _client().brpop(QUEUE_KEY, timeout=5)
|
||||||
|
if raw:
|
||||||
|
await process(json.loads(raw[1])["item_id"])
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("upload worker error: %s", exc)
|
||||||
|
await asyncio.sleep(2)
|
||||||
Loading…
Add table
Add a link
Reference in a new issue