feat: persist and resume ZIP import jobs
This commit is contained in:
parent
d35a88ed66
commit
49f3f4b133
4 changed files with 75 additions and 0 deletions
|
|
@ -2,6 +2,7 @@ import { useEffect } from 'react'
|
||||||
import toast from 'react-hot-toast'
|
import toast from 'react-hot-toast'
|
||||||
import { listUploads } from '@/services/uploadStore'
|
import { listUploads } from '@/services/uploadStore'
|
||||||
import { resumeStoredUpload } from '@/services/uploadQueue'
|
import { resumeStoredUpload } from '@/services/uploadQueue'
|
||||||
|
import { resumeStoredZip } from '@/services/uploadQueue'
|
||||||
|
|
||||||
let started = false
|
let started = false
|
||||||
|
|
||||||
|
|
@ -13,6 +14,11 @@ export function UploadResumeBootstrap() {
|
||||||
const jobs = await listUploads()
|
const jobs = await listUploads()
|
||||||
await Promise.all(jobs.map(async (job) => {
|
await Promise.all(jobs.map(async (job) => {
|
||||||
try {
|
try {
|
||||||
|
if (job.kind === 'zip' || job.kind === 'zip-import') {
|
||||||
|
await resumeStoredZip(job, (message) => toast(message))
|
||||||
|
toast.success(`ZIP đã hoàn tất: ${job.files[0]?.name ?? 'dataset'}`)
|
||||||
|
return
|
||||||
|
}
|
||||||
await resumeStoredUpload(job, (done, total) => {
|
await resumeStoredUpload(job, (done, total) => {
|
||||||
if (done === 0 || done === total) return
|
if (done === 0 || done === total) return
|
||||||
toast(`Đang tiếp tục upload ${done}/${total}: ${job.files[0]?.name ?? ''}`)
|
toast(`Đang tiếp tục upload ${done}/${total}: ${job.files[0]?.name ?? ''}`)
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import {
|
||||||
type UploadTask,
|
type UploadTask,
|
||||||
} from '@/services/uploadQueue'
|
} from '@/services/uploadQueue'
|
||||||
import { api } from '@/services/api'
|
import { api } from '@/services/api'
|
||||||
|
import { saveUpload } from '@/services/uploadStore'
|
||||||
import { API_BASE_URL, STORAGE_KEYS } from '@/app/constants'
|
import { API_BASE_URL, STORAGE_KEYS } from '@/app/constants'
|
||||||
import { getStorageItem } from '@/utils/storage'
|
import { getStorageItem } from '@/utils/storage'
|
||||||
import {
|
import {
|
||||||
|
|
@ -217,6 +218,15 @@ export const DataTab: React.FC<DataTabProps> = ({
|
||||||
`/projects/${project.id}/imports/r2-zip`,
|
`/projects/${project.id}/imports/r2-zip`,
|
||||||
{ r2_key: r2Key }
|
{ r2_key: r2Key }
|
||||||
)
|
)
|
||||||
|
await saveUpload({
|
||||||
|
id: `zip:${project.id}:${zipFile.name}:${zipFile.size}:${zipFile.lastModified}`,
|
||||||
|
projectId: project.id,
|
||||||
|
jobId: job.id,
|
||||||
|
files: [zipFile],
|
||||||
|
r2Key,
|
||||||
|
kind: 'zip-import',
|
||||||
|
createdAt: Date.now(),
|
||||||
|
})
|
||||||
toast.success('Zip đã lên kho, server bắt đầu nhập — bạn cứ làm việc tiếp')
|
toast.success('Zip đã lên kho, server bắt đầu nhập — bạn cứ làm việc tiếp')
|
||||||
pollServerImport(job.id)
|
pollServerImport(job.id)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
|
||||||
|
|
@ -147,6 +147,10 @@ function mpFingerprint(file: File): string {
|
||||||
return `${MP_RESUME_PREFIX}${file.name}-${file.size}-${file.lastModified}`
|
return `${MP_RESUME_PREFIX}${file.name}-${file.size}-${file.lastModified}`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function mpStoreId(projectId: string, file: File): string {
|
||||||
|
return `zip:${projectId}:${file.name}:${file.size}:${file.lastModified}`
|
||||||
|
}
|
||||||
|
|
||||||
function putPart(
|
function putPart(
|
||||||
url: string,
|
url: string,
|
||||||
blob: Blob,
|
blob: Blob,
|
||||||
|
|
@ -175,6 +179,9 @@ export async function uploadFileMultipart(
|
||||||
file: File,
|
file: File,
|
||||||
onProgress?: (ratio: number, uploadedBytes: number) => void
|
onProgress?: (ratio: number, uploadedBytes: number) => void
|
||||||
): Promise<{ r2Key: string }> {
|
): Promise<{ r2Key: string }> {
|
||||||
|
const storeId = mpStoreId(projectId, file)
|
||||||
|
await prepareUploadStorage(file.size)
|
||||||
|
await saveUpload({ id: storeId, projectId, jobId: '', files: [file], createdAt: Date.now(), kind: 'zip' })
|
||||||
const totalParts = Math.max(1, Math.ceil(file.size / MP_PART_SIZE))
|
const totalParts = Math.max(1, Math.ceil(file.size / MP_PART_SIZE))
|
||||||
const saved = (() => {
|
const saved = (() => {
|
||||||
try {
|
try {
|
||||||
|
|
@ -398,3 +405,53 @@ export async function resumeStoredUpload(
|
||||||
}
|
}
|
||||||
await deleteUpload(stored.id)
|
await deleteUpload(stored.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function resumeImportJob(projectId: string, jobId: string, onProgress?: (done: number, total: number) => void) {
|
||||||
|
const token = getStorageItem<string | null>(STORAGE_KEYS.ACCESS_TOKEN, null)
|
||||||
|
const response = await fetch(`${API_BASE_URL}/projects/${projectId}/imports/${jobId}/events`, {
|
||||||
|
headers: token ? { Authorization: `Bearer ${token}` } : undefined,
|
||||||
|
})
|
||||||
|
if (!response.ok || !response.body) throw new Error(`Không kết nối được SSE import (${response.status})`)
|
||||||
|
const reader = response.body.getReader()
|
||||||
|
const decoder = new TextDecoder()
|
||||||
|
let buffer = ''
|
||||||
|
try {
|
||||||
|
for (;;) {
|
||||||
|
const chunk = await reader.read()
|
||||||
|
if (chunk.done) break
|
||||||
|
buffer += decoder.decode(chunk.value, { stream: true })
|
||||||
|
const events = buffer.split('\n\n')
|
||||||
|
buffer = events.pop() ?? ''
|
||||||
|
for (const event of events) {
|
||||||
|
const line = event.split('\n').find((value) => value.startsWith('data:'))
|
||||||
|
if (!line) continue
|
||||||
|
const data = JSON.parse(line.slice(5)) as { status: string; done_files: number; total_files: number; error?: string }
|
||||||
|
onProgress?.(data.done_files, data.total_files)
|
||||||
|
if (data.status === 'done') return
|
||||||
|
if (data.status === 'error') throw new Error(data.error || 'Import ZIP thất bại')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await reader.cancel()
|
||||||
|
}
|
||||||
|
throw new Error('SSE import đã đóng trước khi hoàn tất')
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resumeStoredZip(stored: StoredUpload, onProgress?: (message: string) => void) {
|
||||||
|
const file = stored.files[0]
|
||||||
|
if (!file) throw new Error('Không còn file ZIP để resume')
|
||||||
|
if (stored.kind === 'zip-import' && stored.jobId) {
|
||||||
|
await resumeImportJob(stored.projectId, stored.jobId, (done, total) =>
|
||||||
|
onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`))
|
||||||
|
await deleteUpload(stored.id)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const { r2Key } = await uploadFileMultipart(stored.projectId, file, (ratio) =>
|
||||||
|
onProgress?.(`Đang resume ZIP ${Math.round(ratio * 100)}%`))
|
||||||
|
const { data: job } = await api.post<{ id: string }>(
|
||||||
|
`/projects/${stored.projectId}/imports/r2-zip`, { r2_key: r2Key })
|
||||||
|
await saveUpload({ ...stored, kind: 'zip-import', jobId: job.id, r2Key })
|
||||||
|
await resumeImportJob(stored.projectId, job.id, (done, total) =>
|
||||||
|
onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`))
|
||||||
|
await deleteUpload(stored.id)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ export interface StoredUpload {
|
||||||
jobId: string
|
jobId: string
|
||||||
files: File[]
|
files: File[]
|
||||||
createdAt: number
|
createdAt: number
|
||||||
|
kind?: 'images' | 'zip' | 'zip-import'
|
||||||
|
r2Key?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
const DB_NAME = 'danable-upload-queue'
|
const DB_NAME = 'danable-upload-queue'
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue