76 lines
2.6 KiB
Python
76 lines
2.6 KiB
Python
"""Redis-backed worker for turning uploaded objects into Image rows."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
from sqlalchemy import select
|
|
from shared.logger import get_logger
|
|
from src.core.config import get_settings
|
|
|
|
log = get_logger("danagble.uploads")
|
|
QUEUE_KEY = "danagble:upload-items"
|
|
_redis = None
|
|
|
|
|
|
def _client():
|
|
global _redis
|
|
if _redis is None:
|
|
import redis.asyncio as aioredis
|
|
_redis = aioredis.from_url(get_settings().redis_url, decode_responses=True,
|
|
socket_connect_timeout=1, socket_timeout=10)
|
|
return _redis
|
|
|
|
|
|
async def enqueue(item_id: str) -> None:
|
|
await _client().lpush(QUEUE_KEY, json.dumps({"item_id": item_id}))
|
|
|
|
|
|
async def process(item_id: str) -> None:
|
|
from models import Image, UploadItem, UploadJob
|
|
from src.core.db import get_session_factory
|
|
factory = get_session_factory()
|
|
async with factory() as db:
|
|
item = await db.get(UploadItem, item_id)
|
|
if item is None or item.status != "uploaded":
|
|
return
|
|
job = await db.get(UploadJob, item.job_id)
|
|
if job is None:
|
|
return
|
|
image = Image(project_id=item.project_id, file_name=item.file_name,
|
|
r2_key=item.r2_key, width=item.width, height=item.height,
|
|
status="unassigned", flags={})
|
|
db.add(image)
|
|
item.status = "done"
|
|
item.image_id = image.id
|
|
job.processed_files += 1
|
|
if job.processed_files + job.failed_files >= job.total_files:
|
|
job.status = "done" if job.failed_files == 0 else "partial_error"
|
|
await db.commit()
|
|
await _client().publish(f"danagble:upload:{job.id}", json.dumps({
|
|
"jobId": job.id, "status": job.status,
|
|
"processedFiles": job.processed_files,
|
|
"totalFiles": job.total_files,
|
|
}))
|
|
|
|
|
|
async def run_forever() -> None:
|
|
log.info("upload worker started")
|
|
from models import UploadItem
|
|
from src.core.db import get_session_factory
|
|
factory = get_session_factory()
|
|
async with factory() as db:
|
|
rows = (await db.execute(select(UploadItem).where(
|
|
UploadItem.status == "uploaded"))).scalars().all()
|
|
for row in rows:
|
|
await enqueue(row.id)
|
|
while True:
|
|
try:
|
|
raw = await _client().brpop(QUEUE_KEY, timeout=5)
|
|
if raw:
|
|
await process(json.loads(raw[1])["item_id"])
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
log.warning("upload worker error: %s", exc)
|
|
await asyncio.sleep(2)
|