feat: persist and resume ZIP import jobs
This commit is contained in:
parent
d35a88ed66
commit
49f3f4b133
4 changed files with 75 additions and 0 deletions
|
|
@ -2,6 +2,7 @@ import { useEffect } from 'react'
|
|||
import toast from 'react-hot-toast'
|
||||
import { listUploads } from '@/services/uploadStore'
|
||||
import { resumeStoredUpload } from '@/services/uploadQueue'
|
||||
import { resumeStoredZip } from '@/services/uploadQueue'
|
||||
|
||||
let started = false
|
||||
|
||||
|
|
@ -13,6 +14,11 @@ export function UploadResumeBootstrap() {
|
|||
const jobs = await listUploads()
|
||||
await Promise.all(jobs.map(async (job) => {
|
||||
try {
|
||||
if (job.kind === 'zip' || job.kind === 'zip-import') {
|
||||
await resumeStoredZip(job, (message) => toast(message))
|
||||
toast.success(`ZIP đã hoàn tất: ${job.files[0]?.name ?? 'dataset'}`)
|
||||
return
|
||||
}
|
||||
await resumeStoredUpload(job, (done, total) => {
|
||||
if (done === 0 || done === total) return
|
||||
toast(`Đang tiếp tục upload ${done}/${total}: ${job.files[0]?.name ?? ''}`)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
type UploadTask,
|
||||
} from '@/services/uploadQueue'
|
||||
import { api } from '@/services/api'
|
||||
import { saveUpload } from '@/services/uploadStore'
|
||||
import { API_BASE_URL, STORAGE_KEYS } from '@/app/constants'
|
||||
import { getStorageItem } from '@/utils/storage'
|
||||
import {
|
||||
|
|
@ -217,6 +218,15 @@ export const DataTab: React.FC<DataTabProps> = ({
|
|||
`/projects/${project.id}/imports/r2-zip`,
|
||||
{ r2_key: r2Key }
|
||||
)
|
||||
await saveUpload({
|
||||
id: `zip:${project.id}:${zipFile.name}:${zipFile.size}:${zipFile.lastModified}`,
|
||||
projectId: project.id,
|
||||
jobId: job.id,
|
||||
files: [zipFile],
|
||||
r2Key,
|
||||
kind: 'zip-import',
|
||||
createdAt: Date.now(),
|
||||
})
|
||||
toast.success('Zip đã lên kho, server bắt đầu nhập — bạn cứ làm việc tiếp')
|
||||
pollServerImport(job.id)
|
||||
} catch (err) {
|
||||
|
|
|
|||
|
|
@ -147,6 +147,10 @@ function mpFingerprint(file: File): string {
|
|||
return `${MP_RESUME_PREFIX}${file.name}-${file.size}-${file.lastModified}`
|
||||
}
|
||||
|
||||
function mpStoreId(projectId: string, file: File): string {
|
||||
return `zip:${projectId}:${file.name}:${file.size}:${file.lastModified}`
|
||||
}
|
||||
|
||||
function putPart(
|
||||
url: string,
|
||||
blob: Blob,
|
||||
|
|
@ -175,6 +179,9 @@ export async function uploadFileMultipart(
|
|||
file: File,
|
||||
onProgress?: (ratio: number, uploadedBytes: number) => void
|
||||
): Promise<{ r2Key: string }> {
|
||||
const storeId = mpStoreId(projectId, file)
|
||||
await prepareUploadStorage(file.size)
|
||||
await saveUpload({ id: storeId, projectId, jobId: '', files: [file], createdAt: Date.now(), kind: 'zip' })
|
||||
const totalParts = Math.max(1, Math.ceil(file.size / MP_PART_SIZE))
|
||||
const saved = (() => {
|
||||
try {
|
||||
|
|
@ -398,3 +405,53 @@ export async function resumeStoredUpload(
|
|||
}
|
||||
await deleteUpload(stored.id)
|
||||
}
|
||||
|
||||
async function resumeImportJob(projectId: string, jobId: string, onProgress?: (done: number, total: number) => void) {
|
||||
const token = getStorageItem<string | null>(STORAGE_KEYS.ACCESS_TOKEN, null)
|
||||
const response = await fetch(`${API_BASE_URL}/projects/${projectId}/imports/${jobId}/events`, {
|
||||
headers: token ? { Authorization: `Bearer ${token}` } : undefined,
|
||||
})
|
||||
if (!response.ok || !response.body) throw new Error(`Không kết nối được SSE import (${response.status})`)
|
||||
const reader = response.body.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
try {
|
||||
for (;;) {
|
||||
const chunk = await reader.read()
|
||||
if (chunk.done) break
|
||||
buffer += decoder.decode(chunk.value, { stream: true })
|
||||
const events = buffer.split('\n\n')
|
||||
buffer = events.pop() ?? ''
|
||||
for (const event of events) {
|
||||
const line = event.split('\n').find((value) => value.startsWith('data:'))
|
||||
if (!line) continue
|
||||
const data = JSON.parse(line.slice(5)) as { status: string; done_files: number; total_files: number; error?: string }
|
||||
onProgress?.(data.done_files, data.total_files)
|
||||
if (data.status === 'done') return
|
||||
if (data.status === 'error') throw new Error(data.error || 'Import ZIP thất bại')
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel()
|
||||
}
|
||||
throw new Error('SSE import đã đóng trước khi hoàn tất')
|
||||
}
|
||||
|
||||
export async function resumeStoredZip(stored: StoredUpload, onProgress?: (message: string) => void) {
|
||||
const file = stored.files[0]
|
||||
if (!file) throw new Error('Không còn file ZIP để resume')
|
||||
if (stored.kind === 'zip-import' && stored.jobId) {
|
||||
await resumeImportJob(stored.projectId, stored.jobId, (done, total) =>
|
||||
onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`))
|
||||
await deleteUpload(stored.id)
|
||||
return
|
||||
}
|
||||
const { r2Key } = await uploadFileMultipart(stored.projectId, file, (ratio) =>
|
||||
onProgress?.(`Đang resume ZIP ${Math.round(ratio * 100)}%`))
|
||||
const { data: job } = await api.post<{ id: string }>(
|
||||
`/projects/${stored.projectId}/imports/r2-zip`, { r2_key: r2Key })
|
||||
await saveUpload({ ...stored, kind: 'zip-import', jobId: job.id, r2Key })
|
||||
await resumeImportJob(stored.projectId, job.id, (done, total) =>
|
||||
onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`))
|
||||
await deleteUpload(stored.id)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ export interface StoredUpload {
|
|||
jobId: string
|
||||
files: File[]
|
||||
createdAt: number
|
||||
kind?: 'images' | 'zip' | 'zip-import'
|
||||
r2Key?: string
|
||||
}
|
||||
|
||||
const DB_NAME = 'danable-upload-queue'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue