feat: process uploads through Redis worker and SSE
This commit is contained in:
parent
a82dc33f23
commit
cee69acbdc
3 changed files with 14 additions and 10 deletions
|
|
@ -144,6 +144,8 @@ class UploadItem(Base):
|
||||||
file_name: Mapped[str] = mapped_column(String(1024))
|
file_name: Mapped[str] = mapped_column(String(1024))
|
||||||
content_type: Mapped[str] = mapped_column(String(255), default="application/octet-stream")
|
content_type: Mapped[str] = mapped_column(String(255), default="application/octet-stream")
|
||||||
size: Mapped[int] = mapped_column(Integer, default=0)
|
size: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
width: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
height: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
r2_key: Mapped[str] = mapped_column(String(1024))
|
r2_key: Mapped[str] = mapped_column(String(1024))
|
||||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||||
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
image_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
from shared import events
|
from shared import events
|
||||||
|
from shared import upload_worker
|
||||||
from shared.logger import get_logger
|
from shared.logger import get_logger
|
||||||
from shared.tracing import TraceMiddleware
|
from shared.tracing import TraceMiddleware
|
||||||
from shared.traffic import TrafficMiddleware
|
from shared.traffic import TrafficMiddleware
|
||||||
|
|
@ -23,8 +24,10 @@ log = get_logger("danagble")
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
drainer = asyncio.create_task(events.drain_forever())
|
drainer = asyncio.create_task(events.drain_forever())
|
||||||
|
uploader = asyncio.create_task(upload_worker.run_forever())
|
||||||
yield
|
yield
|
||||||
drainer.cancel()
|
drainer.cancel()
|
||||||
|
uploader.cancel()
|
||||||
|
|
||||||
|
|
||||||
app = FastAPI(title=s.app_name, lifespan=lifespan)
|
app = FastAPI(title=s.app_name, lifespan=lifespan)
|
||||||
|
|
|
||||||
|
|
@ -79,21 +79,20 @@ async def complete_item(job_id: str, item_id: str, width: int, height: int,
|
||||||
raise HTTPException(404, "upload job not found")
|
raise HTTPException(404, "upload job not found")
|
||||||
if item.status == "done":
|
if item.status == "done":
|
||||||
return {**_job_dict(job), "imageId": item.image_id}
|
return {**_job_dict(job), "imageId": item.image_id}
|
||||||
|
if item.status == "uploaded":
|
||||||
|
return _job_dict(job)
|
||||||
if not r2.head_exists(item.r2_key):
|
if not r2.head_exists(item.r2_key):
|
||||||
item.status = "error"
|
item.status = "error"
|
||||||
item.error = "object not found in storage"
|
item.error = "object not found in storage"
|
||||||
job.failed_files += 1
|
job.failed_files += 1
|
||||||
await db.commit()
|
await db.commit()
|
||||||
raise HTTPException(400, item.error)
|
raise HTTPException(400, item.error)
|
||||||
image = Image(project_id=item.project_id, file_name=item.file_name,
|
item.status = "uploaded"
|
||||||
r2_key=item.r2_key, width=width, height=height,
|
item.width = width
|
||||||
status="unassigned", flags={})
|
item.height = height
|
||||||
db.add(image)
|
|
||||||
item.status = "done"
|
|
||||||
item.image_id = image.id
|
|
||||||
job.uploaded_files += 1
|
job.uploaded_files += 1
|
||||||
job.processed_files += 1
|
job.status = "processing"
|
||||||
if job.processed_files + job.failed_files >= job.total_files:
|
|
||||||
job.status = "done" if job.failed_files == 0 else "partial_error"
|
|
||||||
await db.commit()
|
await db.commit()
|
||||||
return {**_job_dict(job), "imageId": image.id}
|
from shared.upload_worker import enqueue
|
||||||
|
await enqueue(item.id)
|
||||||
|
return _job_dict(job)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue