feat: process uploads through Redis worker and SSE

This commit is contained in:
hailt 2026-11-12 22:44:22 +07:00
parent a82dc33f23
commit cee69acbdc
3 changed files with 14 additions and 10 deletions

View file

@ -144,6 +144,8 @@ class UploadItem(Base):
file_name: Mapped[str] = mapped_column(String(1024)) file_name: Mapped[str] = mapped_column(String(1024))
content_type: Mapped[str] = mapped_column(String(255), default="application/octet-stream") content_type: Mapped[str] = mapped_column(String(255), default="application/octet-stream")
size: Mapped[int] = mapped_column(Integer, default=0) size: Mapped[int] = mapped_column(Integer, default=0)
width: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
height: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
r2_key: Mapped[str] = mapped_column(String(1024)) r2_key: Mapped[str] = mapped_column(String(1024))
status: Mapped[str] = mapped_column(String(32), default="pending", index=True) status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True) image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)

View file

@ -8,6 +8,7 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from shared import events from shared import events
from shared import upload_worker
from shared.logger import get_logger from shared.logger import get_logger
from shared.tracing import TraceMiddleware from shared.tracing import TraceMiddleware
from shared.traffic import TrafficMiddleware from shared.traffic import TrafficMiddleware
@ -23,8 +24,10 @@ log = get_logger("danagble")
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
drainer = asyncio.create_task(events.drain_forever()) drainer = asyncio.create_task(events.drain_forever())
uploader = asyncio.create_task(upload_worker.run_forever())
yield yield
drainer.cancel() drainer.cancel()
uploader.cancel()
app = FastAPI(title=s.app_name, lifespan=lifespan) app = FastAPI(title=s.app_name, lifespan=lifespan)

View file

@ -79,21 +79,20 @@ async def complete_item(job_id: str, item_id: str, width: int, height: int,
raise HTTPException(404, "upload job not found") raise HTTPException(404, "upload job not found")
if item.status == "done": if item.status == "done":
return {**_job_dict(job), "imageId": item.image_id} return {**_job_dict(job), "imageId": item.image_id}
if item.status == "uploaded":
return _job_dict(job)
if not r2.head_exists(item.r2_key): if not r2.head_exists(item.r2_key):
item.status = "error" item.status = "error"
item.error = "object not found in storage" item.error = "object not found in storage"
job.failed_files += 1 job.failed_files += 1
await db.commit() await db.commit()
raise HTTPException(400, item.error) raise HTTPException(400, item.error)
image = Image(project_id=item.project_id, file_name=item.file_name, item.status = "uploaded"
r2_key=item.r2_key, width=width, height=height, item.width = width
status="unassigned", flags={}) item.height = height
db.add(image)
item.status = "done"
item.image_id = image.id
job.uploaded_files += 1 job.uploaded_files += 1
job.processed_files += 1 job.status = "processing"
if job.processed_files + job.failed_files >= job.total_files:
job.status = "done" if job.failed_files == 0 else "partial_error"
await db.commit() await db.commit()
return {**_job_dict(job), "imageId": image.id} from shared.upload_worker import enqueue
await enqueue(item.id)
return _job_dict(job)