mirror of
https://github.com/danagble/backend.git
synced 2026-11-16 03:00:21 +00:00
feat: add Redis upload worker
This commit is contained in:
parent
cee69acbdc
commit
5a546e4e40
1 changed files with 76 additions and 0 deletions
76
shared/upload_worker.py
Normal file
76
shared/upload_worker.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
"""Redis-backed worker for turning uploaded objects into Image rows."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from sqlalchemy import select
|
||||
from shared.logger import get_logger
|
||||
from src.core.config import get_settings
|
||||
|
||||
log = get_logger("danagble.uploads")
|
||||
QUEUE_KEY = "danagble:upload-items"
|
||||
_redis = None
|
||||
|
||||
|
||||
def _client():
|
||||
global _redis
|
||||
if _redis is None:
|
||||
import redis.asyncio as aioredis
|
||||
_redis = aioredis.from_url(get_settings().redis_url, decode_responses=True,
|
||||
socket_connect_timeout=1, socket_timeout=5)
|
||||
return _redis
|
||||
|
||||
|
||||
async def enqueue(item_id: str) -> None:
|
||||
await _client().lpush(QUEUE_KEY, json.dumps({"item_id": item_id}))
|
||||
|
||||
|
||||
async def process(item_id: str) -> None:
|
||||
from models import Image, UploadItem, UploadJob
|
||||
from src.core.db import get_session_factory
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
item = await db.get(UploadItem, item_id)
|
||||
if item is None or item.status != "uploaded":
|
||||
return
|
||||
job = await db.get(UploadJob, item.job_id)
|
||||
if job is None:
|
||||
return
|
||||
image = Image(project_id=item.project_id, file_name=item.file_name,
|
||||
r2_key=item.r2_key, width=item.width, height=item.height,
|
||||
status="unassigned", flags={})
|
||||
db.add(image)
|
||||
item.status = "done"
|
||||
item.image_id = image.id
|
||||
job.processed_files += 1
|
||||
if job.processed_files + job.failed_files >= job.total_files:
|
||||
job.status = "done" if job.failed_files == 0 else "partial_error"
|
||||
await db.commit()
|
||||
await _client().publish(f"danagble:upload:{job.id}", json.dumps({
|
||||
"jobId": job.id, "status": job.status,
|
||||
"processedFiles": job.processed_files,
|
||||
"totalFiles": job.total_files,
|
||||
}))
|
||||
|
||||
|
||||
async def run_forever() -> None:
|
||||
log.info("upload worker started")
|
||||
from models import UploadItem
|
||||
from src.core.db import get_session_factory
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
rows = (await db.execute(select(UploadItem).where(
|
||||
UploadItem.status == "uploaded"))).scalars().all()
|
||||
for row in rows:
|
||||
await enqueue(row.id)
|
||||
while True:
|
||||
try:
|
||||
raw = await _client().brpop(QUEUE_KEY, timeout=5)
|
||||
if raw:
|
||||
await process(json.loads(raw[1])["item_id"])
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.warning("upload worker error: %s", exc)
|
||||
await asyncio.sleep(2)
|
||||
Loading…
Add table
Add a link
Reference in a new issue