feat: persist and resume ZIP import jobs

This commit is contained in:
hailt 2026-11-12 22:54:59 +07:00
parent d35a88ed66
commit 49f3f4b133
4 changed files with 75 additions and 0 deletions

View file

@ -2,6 +2,7 @@ import { useEffect } from 'react'
import toast from 'react-hot-toast' import toast from 'react-hot-toast'
import { listUploads } from '@/services/uploadStore' import { listUploads } from '@/services/uploadStore'
import { resumeStoredUpload } from '@/services/uploadQueue' import { resumeStoredUpload } from '@/services/uploadQueue'
import { resumeStoredZip } from '@/services/uploadQueue'
let started = false let started = false
@ -13,6 +14,11 @@ export function UploadResumeBootstrap() {
const jobs = await listUploads() const jobs = await listUploads()
await Promise.all(jobs.map(async (job) => { await Promise.all(jobs.map(async (job) => {
try { try {
if (job.kind === 'zip' || job.kind === 'zip-import') {
await resumeStoredZip(job, (message) => toast(message))
toast.success(`ZIP đã hoàn tất: ${job.files[0]?.name ?? 'dataset'}`)
return
}
await resumeStoredUpload(job, (done, total) => { await resumeStoredUpload(job, (done, total) => {
if (done === 0 || done === total) return if (done === 0 || done === total) return
toast(`Đang tiếp tục upload ${done}/${total}: ${job.files[0]?.name ?? ''}`) toast(`Đang tiếp tục upload ${done}/${total}: ${job.files[0]?.name ?? ''}`)

View file

@ -9,6 +9,7 @@ import {
type UploadTask, type UploadTask,
} from '@/services/uploadQueue' } from '@/services/uploadQueue'
import { api } from '@/services/api' import { api } from '@/services/api'
import { saveUpload } from '@/services/uploadStore'
import { API_BASE_URL, STORAGE_KEYS } from '@/app/constants' import { API_BASE_URL, STORAGE_KEYS } from '@/app/constants'
import { getStorageItem } from '@/utils/storage' import { getStorageItem } from '@/utils/storage'
import { import {
@ -217,6 +218,15 @@ export const DataTab: React.FC<DataTabProps> = ({
`/projects/${project.id}/imports/r2-zip`, `/projects/${project.id}/imports/r2-zip`,
{ r2_key: r2Key } { r2_key: r2Key }
) )
await saveUpload({
id: `zip:${project.id}:${zipFile.name}:${zipFile.size}:${zipFile.lastModified}`,
projectId: project.id,
jobId: job.id,
files: [zipFile],
r2Key,
kind: 'zip-import',
createdAt: Date.now(),
})
toast.success('Zip đã lên kho, server bắt đầu nhập — bạn cứ làm việc tiếp') toast.success('Zip đã lên kho, server bắt đầu nhập — bạn cứ làm việc tiếp')
pollServerImport(job.id) pollServerImport(job.id)
} catch (err) { } catch (err) {

View file

@ -147,6 +147,10 @@ function mpFingerprint(file: File): string {
return `${MP_RESUME_PREFIX}${file.name}-${file.size}-${file.lastModified}` return `${MP_RESUME_PREFIX}${file.name}-${file.size}-${file.lastModified}`
} }
function mpStoreId(projectId: string, file: File): string {
return `zip:${projectId}:${file.name}:${file.size}:${file.lastModified}`
}
function putPart( function putPart(
url: string, url: string,
blob: Blob, blob: Blob,
@ -175,6 +179,9 @@ export async function uploadFileMultipart(
file: File, file: File,
onProgress?: (ratio: number, uploadedBytes: number) => void onProgress?: (ratio: number, uploadedBytes: number) => void
): Promise<{ r2Key: string }> { ): Promise<{ r2Key: string }> {
const storeId = mpStoreId(projectId, file)
await prepareUploadStorage(file.size)
await saveUpload({ id: storeId, projectId, jobId: '', files: [file], createdAt: Date.now(), kind: 'zip' })
const totalParts = Math.max(1, Math.ceil(file.size / MP_PART_SIZE)) const totalParts = Math.max(1, Math.ceil(file.size / MP_PART_SIZE))
const saved = (() => { const saved = (() => {
try { try {
@ -398,3 +405,53 @@ export async function resumeStoredUpload(
} }
await deleteUpload(stored.id) await deleteUpload(stored.id)
} }
async function resumeImportJob(projectId: string, jobId: string, onProgress?: (done: number, total: number) => void) {
const token = getStorageItem<string | null>(STORAGE_KEYS.ACCESS_TOKEN, null)
const response = await fetch(`${API_BASE_URL}/projects/${projectId}/imports/${jobId}/events`, {
headers: token ? { Authorization: `Bearer ${token}` } : undefined,
})
if (!response.ok || !response.body) throw new Error(`Không kết nối được SSE import (${response.status})`)
const reader = response.body.getReader()
const decoder = new TextDecoder()
let buffer = ''
try {
for (;;) {
const chunk = await reader.read()
if (chunk.done) break
buffer += decoder.decode(chunk.value, { stream: true })
const events = buffer.split('\n\n')
buffer = events.pop() ?? ''
for (const event of events) {
const line = event.split('\n').find((value) => value.startsWith('data:'))
if (!line) continue
const data = JSON.parse(line.slice(5)) as { status: string; done_files: number; total_files: number; error?: string }
onProgress?.(data.done_files, data.total_files)
if (data.status === 'done') return
if (data.status === 'error') throw new Error(data.error || 'Import ZIP thất bại')
}
}
} finally {
await reader.cancel()
}
throw new Error('SSE import đã đóng trước khi hoàn tất')
}
export async function resumeStoredZip(stored: StoredUpload, onProgress?: (message: string) => void) {
const file = stored.files[0]
if (!file) throw new Error('Không còn file ZIP để resume')
if (stored.kind === 'zip-import' && stored.jobId) {
await resumeImportJob(stored.projectId, stored.jobId, (done, total) =>
onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`))
await deleteUpload(stored.id)
return
}
const { r2Key } = await uploadFileMultipart(stored.projectId, file, (ratio) =>
onProgress?.(`Đang resume ZIP ${Math.round(ratio * 100)}%`))
const { data: job } = await api.post<{ id: string }>(
`/projects/${stored.projectId}/imports/r2-zip`, { r2_key: r2Key })
await saveUpload({ ...stored, kind: 'zip-import', jobId: job.id, r2Key })
await resumeImportJob(stored.projectId, job.id, (done, total) =>
onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`))
await deleteUpload(stored.id)
}

View file

@ -4,6 +4,8 @@ export interface StoredUpload {
jobId: string jobId: string
files: File[] files: File[]
createdAt: number createdAt: number
kind?: 'images' | 'zip' | 'zip-import'
r2Key?: string
} }
const DB_NAME = 'danable-upload-queue' const DB_NAME = 'danable-upload-queue'