472 lines
16 KiB
TypeScript
472 lines
16 KiB
TypeScript
import { api } from './api'
|
|
import { deleteUpload, prepareUploadStorage, saveUpload, type StoredUpload } from './uploadStore'
|
|
import { API_BASE_URL, STORAGE_KEYS } from '@/app/constants'
|
|
import { getStorageItem } from '@/utils/storage'
|
|
import { importCocoFile, importYoloFiles } from './datasetImportService'
|
|
|
|
export type UploadTaskStatus = 'pending' | 'uploading' | 'done' | 'error' | 'skipped'
|
|
|
|
export interface UploadTask {
|
|
fileName: string
|
|
status: UploadTaskStatus
|
|
progress: number
|
|
error?: string
|
|
imageId?: string
|
|
}
|
|
|
|
export interface BatchUploadResult {
|
|
done: { fileName: string; imageId: string }[]
|
|
failed: { fileName: string; error: string }[]
|
|
skipped: string[]
|
|
}
|
|
|
|
interface PresignSlot {
|
|
id: string
|
|
fileName: string
|
|
putUrl: string
|
|
r2Key: string
|
|
}
|
|
|
|
const CONCURRENCY = 4
|
|
const MAX_RETRIES = 2
|
|
|
|
function putWithProgress(
|
|
url: string,
|
|
file: File,
|
|
contentType: string,
|
|
onProgress: (ratio: number) => void
|
|
): Promise<void> {
|
|
// fetch has no upload progress; XHR is used for this single call only.
|
|
return new Promise((resolve, reject) => {
|
|
const xhr = new XMLHttpRequest()
|
|
xhr.open('PUT', url)
|
|
xhr.setRequestHeader('Content-Type', contentType)
|
|
xhr.upload.onprogress = (e) => {
|
|
if (e.lengthComputable) onProgress(e.loaded / e.total)
|
|
}
|
|
xhr.onload = () =>
|
|
xhr.status >= 200 && xhr.status < 300
|
|
? resolve()
|
|
: reject(new Error(`Tải lên kho lưu trữ thất bại (${xhr.status})`))
|
|
xhr.onerror = () => reject(new Error('Lỗi mạng khi tải lên'))
|
|
xhr.send(file)
|
|
})
|
|
}
|
|
|
|
async function readDimensions(file: File): Promise<{ width: number; height: number }> {
|
|
try {
|
|
const bmp = await createImageBitmap(file)
|
|
const dims = { width: bmp.width, height: bmp.height }
|
|
bmp.close()
|
|
return dims
|
|
} catch {
|
|
return { width: 0, height: 0 }
|
|
}
|
|
}
|
|
|
|
// Recursively collect files from a drop (files and/or folders) preserving names.
|
|
export async function collectDroppedFiles(dt: DataTransfer): Promise<File[]> {
|
|
const items = Array.from(dt.items).filter((i) => i.kind === 'file')
|
|
const entries = items
|
|
.map((i) => (typeof i.webkitGetAsEntry === 'function' ? i.webkitGetAsEntry() : null))
|
|
.filter((e): e is FileSystemEntry => e !== null)
|
|
if (entries.length === 0 || entries.every((e) => e.isFile)) {
|
|
return Array.from(dt.files)
|
|
}
|
|
const out: File[] = []
|
|
const walk = async (entry: FileSystemEntry, prefix: string): Promise<void> => {
|
|
if (entry.isFile) {
|
|
const blob = await new Promise<File>((resolve, reject) =>
|
|
(entry as FileSystemFileEntry).file(resolve, reject)
|
|
)
|
|
out.push(new File([blob], prefix + blob.name, { type: blob.type }))
|
|
return
|
|
}
|
|
if (entry.isDirectory) {
|
|
const reader = (entry as FileSystemDirectoryEntry).createReader()
|
|
for (;;) {
|
|
const batch: FileSystemEntry[] = await new Promise((resolve, reject) =>
|
|
reader.readEntries(resolve, reject)
|
|
)
|
|
if (batch.length === 0) break
|
|
for (const child of batch) await walk(child, `${prefix}${entry.name}/`)
|
|
}
|
|
}
|
|
}
|
|
for (const entry of entries) await walk(entry, '')
|
|
return out
|
|
}
|
|
|
|
// ---- Resumable multipart upload for single big files (100MB..GBs) ----
|
|
// Parts go straight to R2 in parallel with per-part retry; ETags persist in
|
|
// localStorage so a dropped network resumes instead of restarting.
|
|
const MP_PART_SIZE = 8 * 1024 * 1024
|
|
const MP_CONCURRENCY = 4
|
|
const MP_RETRIES = 2
|
|
const MP_RESUME_PREFIX = 'danable:mp:'
|
|
|
|
async function waitForProcessedItem(projectId: string, jobId: string, itemId: string): Promise<string> {
|
|
const token = getStorageItem<string | null>(STORAGE_KEYS.ACCESS_TOKEN, null)
|
|
const response = await fetch(`${API_BASE_URL}/projects/${projectId}/upload-jobs/${jobId}/events`, {
|
|
headers: token ? { Authorization: `Bearer ${token}` } : undefined,
|
|
})
|
|
if (!response.ok || !response.body) throw new Error(`Không kết nối được SSE (${response.status})`)
|
|
const reader = response.body.getReader()
|
|
const decoder = new TextDecoder()
|
|
let buffer = ''
|
|
try {
|
|
for (;;) {
|
|
const chunk = await reader.read()
|
|
if (chunk.done) break
|
|
buffer += decoder.decode(chunk.value, { stream: true })
|
|
const events = buffer.split('\n\n')
|
|
buffer = events.pop() ?? ''
|
|
for (const event of events) {
|
|
if (!event.includes('data:')) continue
|
|
const { data } = await api.get<{ items: Array<{ id: string; status: string; imageId?: string; error?: string }> }>(
|
|
`/projects/${projectId}/upload-jobs/${jobId}`
|
|
)
|
|
const item = data.items.find((candidate) => candidate.id === itemId)
|
|
if (item?.status === 'done' && item.imageId) return item.imageId
|
|
if (item?.status === 'error') throw new Error(item.error || 'Xử lý ảnh thất bại')
|
|
}
|
|
}
|
|
} finally {
|
|
await reader.cancel()
|
|
}
|
|
throw new Error('SSE upload worker đã đóng trước khi xử lý xong')
|
|
}
|
|
|
|
interface MpResume {
|
|
key: string
|
|
uploadId: string
|
|
etags: Record<number, string>
|
|
totalParts: number
|
|
}
|
|
|
|
function mpFingerprint(file: File): string {
|
|
return `${MP_RESUME_PREFIX}${file.name}-${file.size}-${file.lastModified}`
|
|
}
|
|
|
|
function mpStoreId(projectId: string, file: File): string {
|
|
return `zip:${projectId}:${file.name}:${file.size}:${file.lastModified}`
|
|
}
|
|
|
|
function putPart(
|
|
url: string,
|
|
blob: Blob,
|
|
onProgress: (loaded: number) => void
|
|
): Promise<string> {
|
|
return new Promise((resolve, reject) => {
|
|
const xhr = new XMLHttpRequest()
|
|
xhr.open('PUT', url)
|
|
xhr.upload.onprogress = (e) => {
|
|
if (e.lengthComputable) onProgress(e.loaded)
|
|
}
|
|
xhr.onload = () => {
|
|
if (xhr.status >= 200 && xhr.status < 300) {
|
|
resolve(xhr.getResponseHeader('ETag') ?? '')
|
|
} else {
|
|
reject(new Error(`Part thất bại (${xhr.status})`))
|
|
}
|
|
}
|
|
xhr.onerror = () => reject(new Error('Lỗi mạng khi tải part'))
|
|
xhr.send(blob)
|
|
})
|
|
}
|
|
|
|
export async function uploadFileMultipart(
|
|
projectId: string,
|
|
file: File,
|
|
onProgress?: (ratio: number, uploadedBytes: number) => void
|
|
): Promise<{ r2Key: string }> {
|
|
const storeId = mpStoreId(projectId, file)
|
|
await prepareUploadStorage(file.size)
|
|
await saveUpload({ id: storeId, projectId, jobId: '', files: [file], createdAt: Date.now(), kind: 'zip' })
|
|
const totalParts = Math.max(1, Math.ceil(file.size / MP_PART_SIZE))
|
|
const saved = (() => {
|
|
try {
|
|
const raw = localStorage.getItem(mpFingerprint(file))
|
|
return raw ? (JSON.parse(raw) as MpResume) : null
|
|
} catch {
|
|
return null
|
|
}
|
|
})()
|
|
|
|
let key: string
|
|
let uploadId: string
|
|
const etags: Record<number, string> = { ...(saved?.etags ?? {}) }
|
|
if (saved && saved.totalParts === totalParts) {
|
|
// Resume: reuse key + uploadId, skip parts that already have ETags.
|
|
key = saved.key
|
|
uploadId = saved.uploadId
|
|
const { data: adopted } = await api.post<{ jobId: string }>(
|
|
`/projects/${projectId}/uploads/multipart/adopt`,
|
|
{ fileName: file.name, contentType: file.type || 'application/octet-stream', key, uploadId }
|
|
)
|
|
await saveUpload({ id: storeId, projectId, jobId: adopted.jobId, files: [file],
|
|
createdAt: Date.now(), kind: 'zip', r2Key: key })
|
|
} else {
|
|
const { data } = await api.post<{ key: string; uploadId: string; jobId?: string }>(
|
|
`/projects/${projectId}/uploads/multipart`,
|
|
{ fileName: file.name, contentType: file.type || 'application/octet-stream' }
|
|
)
|
|
key = data.key
|
|
uploadId = data.uploadId
|
|
await saveUpload({ id: storeId, projectId, jobId: data.jobId ?? '', files: [file],
|
|
createdAt: Date.now(), kind: 'zip', r2Key: key })
|
|
}
|
|
const persist = () => {
|
|
try {
|
|
localStorage.setItem(
|
|
mpFingerprint(file),
|
|
JSON.stringify({ key, uploadId, etags, totalParts })
|
|
)
|
|
} catch {
|
|
// Private mode: resume simply unavailable.
|
|
}
|
|
}
|
|
persist()
|
|
|
|
const { data: partUrls } = await api.post<{ urls: Record<number, string> }>(
|
|
`/projects/${projectId}/uploads/multipart/parts`,
|
|
{ key, uploadId, numbers: Array.from({ length: totalParts }, (_, i) => i + 1) }
|
|
)
|
|
|
|
const loaded = new Array<number>(totalParts).fill(0)
|
|
const report = () => {
|
|
const sum = loaded.reduce((a, b) => a + b, 0)
|
|
onProgress?.(sum / Math.max(1, file.size), sum)
|
|
}
|
|
let cursor = 0
|
|
const numbers = Array.from({ length: totalParts }, (_, i) => i + 1)
|
|
const worker = async (): Promise<void> => {
|
|
for (;;) {
|
|
const n = numbers[cursor++]
|
|
if (n === undefined) return
|
|
if (etags[n]) continue
|
|
const start = (n - 1) * MP_PART_SIZE
|
|
const blob = file.slice(start, start + MP_PART_SIZE)
|
|
let done = false
|
|
for (let attempt = 0; attempt <= MP_RETRIES && !done; attempt++) {
|
|
try {
|
|
etags[n] = await putPart(partUrls.urls[n]!, blob, (ld) => {
|
|
loaded[n - 1] = ld
|
|
report()
|
|
})
|
|
loaded[n - 1] = blob.size
|
|
report()
|
|
persist()
|
|
done = true
|
|
} catch (err) {
|
|
if (attempt === MP_RETRIES) throw err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
await Promise.all(Array.from({ length: Math.min(MP_CONCURRENCY, totalParts) }, worker))
|
|
|
|
await api.post(`/projects/${projectId}/uploads/multipart/complete`, {
|
|
key,
|
|
uploadId,
|
|
parts: Object.entries(etags).map(([num, ETag]) => ({ ETag, PartNumber: Number(num) })),
|
|
})
|
|
try {
|
|
localStorage.removeItem(mpFingerprint(file))
|
|
} catch {
|
|
// ignore
|
|
}
|
|
onProgress?.(1, file.size)
|
|
return { r2Key: key }
|
|
}
|
|
|
|
export async function uploadFilesBatch(
|
|
projectId: string,
|
|
files: File[],
|
|
onUpdate?: (tasks: UploadTask[]) => void
|
|
): Promise<BatchUploadResult> {
|
|
const seen = new Set<string>()
|
|
const tasks: UploadTask[] = files.map((f) => {
|
|
if (seen.has(f.name)) {
|
|
return { fileName: f.name, status: 'skipped', progress: 0 }
|
|
}
|
|
seen.add(f.name)
|
|
return { fileName: f.name, status: 'pending', progress: 0 }
|
|
})
|
|
const emit = () => onUpdate?.(tasks.map((t) => ({ ...t })))
|
|
emit()
|
|
const pending = files.filter((f) => tasks.find((t) => t.fileName === f.name)?.status === 'pending')
|
|
if (pending.length === 0) {
|
|
return { done: [], failed: [], skipped: tasks.map((t) => t.fileName) }
|
|
}
|
|
|
|
await prepareUploadStorage(pending.reduce((sum, file) => sum + file.size, 0))
|
|
const { data: created } = await api.post<{
|
|
job: { id: string }
|
|
items: PresignSlot[]
|
|
}>(`/projects/${projectId}/upload-jobs`, {
|
|
kind: 'images',
|
|
files: pending.map((file) => ({
|
|
fileName: file.name,
|
|
contentType: file.type || 'image/jpeg',
|
|
size: file.size,
|
|
})),
|
|
})
|
|
const jobId = created.job.id
|
|
await saveUpload({
|
|
id: jobId,
|
|
projectId,
|
|
jobId,
|
|
files: pending,
|
|
createdAt: Date.now(),
|
|
})
|
|
|
|
const slots = new Map<string, PresignSlot>()
|
|
created.items.forEach((s) => slots.set(s.fileName, s))
|
|
|
|
const result: BatchUploadResult = { done: [], failed: [], skipped: [] }
|
|
tasks.forEach((t) => {
|
|
if (t.status === 'skipped') result.skipped.push(t.fileName)
|
|
})
|
|
|
|
let cursor = 0
|
|
const queue = pending.slice()
|
|
const worker = async (): Promise<void> => {
|
|
for (;;) {
|
|
const file = queue[cursor++]
|
|
if (!file) return
|
|
const task = tasks.find((t) => t.fileName === file.name)!
|
|
const slot = slots.get(file.name)
|
|
if (!slot) {
|
|
task.status = 'error'
|
|
task.error = 'Không lấy được URL tải lên'
|
|
result.failed.push({ fileName: file.name, error: task.error })
|
|
emit()
|
|
continue
|
|
}
|
|
task.status = 'uploading'
|
|
emit()
|
|
let ok = false
|
|
for (let attempt = 0; attempt <= MAX_RETRIES && !ok; attempt++) {
|
|
try {
|
|
const dims = await readDimensions(file)
|
|
await putWithProgress(slot.putUrl, file, file.type || 'image/jpeg', (r) => {
|
|
task.progress = r
|
|
emit()
|
|
})
|
|
await api.post<{ processedFiles: number }>(
|
|
`/projects/${projectId}/upload-jobs/${jobId}/items/${slot.id}/complete`,
|
|
{ width: dims.width, height: dims.height }
|
|
)
|
|
const imageId = await waitForProcessedItem(projectId, jobId, slot.id)
|
|
task.status = 'done'
|
|
task.progress = 1
|
|
task.imageId = imageId
|
|
result.done.push({ fileName: file.name, imageId })
|
|
ok = true
|
|
} catch (err) {
|
|
task.error = err instanceof Error ? err.message : 'Tải lên thất bại'
|
|
if (attempt === MAX_RETRIES) {
|
|
task.status = 'error'
|
|
result.failed.push({ fileName: file.name, error: task.error })
|
|
}
|
|
} finally {
|
|
emit()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, queue.length) }, worker))
|
|
if (result.failed.length === 0) await deleteUpload(jobId)
|
|
return result
|
|
}
|
|
|
|
export async function resumeStoredUpload(
|
|
stored: StoredUpload,
|
|
onProgress?: (done: number, total: number) => void
|
|
): Promise<void> {
|
|
const { data: job } = await api.get<{
|
|
status: string
|
|
totalFiles: number
|
|
processedFiles: number
|
|
items: Array<{ id: string; fileName: string; status: string; putUrl?: string }>
|
|
}>(`/projects/${stored.projectId}/upload-jobs/${stored.jobId}`)
|
|
if (job.status === 'done') {
|
|
await deleteUpload(stored.id)
|
|
return
|
|
}
|
|
const byName = new Map(stored.files.map((file) => [file.name, file]))
|
|
let done = job.processedFiles
|
|
onProgress?.(done, job.totalFiles)
|
|
for (const item of job.items) {
|
|
if (item.status === 'done') continue
|
|
const file = byName.get(item.fileName)
|
|
if (!file || !item.putUrl) throw new Error(`Không tìm thấy file để resume: ${item.fileName}`)
|
|
const dims = await readDimensions(file)
|
|
await putWithProgress(item.putUrl, file, file.type || 'image/jpeg', () => undefined)
|
|
await api.post(
|
|
`/projects/${stored.projectId}/upload-jobs/${stored.jobId}/items/${item.id}/complete`,
|
|
{ width: dims.width, height: dims.height }
|
|
)
|
|
await waitForProcessedItem(stored.projectId, stored.jobId, item.id)
|
|
done += 1
|
|
onProgress?.(done, job.totalFiles)
|
|
}
|
|
await deleteUpload(stored.id)
|
|
}
|
|
|
|
async function resumeImportJob(projectId: string, jobId: string, onProgress?: (done: number, total: number) => void) {
|
|
const token = getStorageItem<string | null>(STORAGE_KEYS.ACCESS_TOKEN, null)
|
|
const response = await fetch(`${API_BASE_URL}/projects/${projectId}/imports/${jobId}/events`, {
|
|
headers: token ? { Authorization: `Bearer ${token}` } : undefined,
|
|
})
|
|
if (!response.ok || !response.body) throw new Error(`Không kết nối được SSE import (${response.status})`)
|
|
const reader = response.body.getReader()
|
|
const decoder = new TextDecoder()
|
|
let buffer = ''
|
|
try {
|
|
for (;;) {
|
|
const chunk = await reader.read()
|
|
if (chunk.done) break
|
|
buffer += decoder.decode(chunk.value, { stream: true })
|
|
const events = buffer.split('\n\n')
|
|
buffer = events.pop() ?? ''
|
|
for (const event of events) {
|
|
const line = event.split('\n').find((value) => value.startsWith('data:'))
|
|
if (!line) continue
|
|
const data = JSON.parse(line.slice(5)) as { status: string; done_files: number; total_files: number; error?: string }
|
|
onProgress?.(data.done_files, data.total_files)
|
|
if (data.status === 'done') return
|
|
if (data.status === 'error') throw new Error(data.error || 'Import ZIP thất bại')
|
|
}
|
|
}
|
|
} finally {
|
|
await reader.cancel()
|
|
}
|
|
throw new Error('SSE import đã đóng trước khi hoàn tất')
|
|
}
|
|
|
|
export async function resumeStoredZip(stored: StoredUpload, onProgress?: (message: string) => void) {
|
|
const file = stored.files[0]
|
|
if (!file) throw new Error('Không còn file ZIP để resume')
|
|
if (stored.kind === 'zip-import' && stored.jobId) {
|
|
await resumeImportJob(stored.projectId, stored.jobId, (done, total) =>
|
|
onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`))
|
|
await deleteUpload(stored.id)
|
|
return
|
|
}
|
|
const { r2Key } = await uploadFileMultipart(stored.projectId, file, (ratio) =>
|
|
onProgress?.(`Đang resume ZIP ${Math.round(ratio * 100)}%`))
|
|
const { data: job } = await api.post<{ id: string }>(
|
|
`/projects/${stored.projectId}/imports/r2-zip`, { r2_key: r2Key })
|
|
await saveUpload({ ...stored, kind: 'zip-import', jobId: job.id, r2Key })
|
|
await resumeImportJob(stored.projectId, job.id, (done, total) =>
|
|
onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`))
|
|
await deleteUpload(stored.id)
|
|
}
|
|
|
|
export async function resumeStoredAnnotationImport(stored: StoredUpload) {
|
|
if (stored.kind === 'coco') await importCocoFile(stored.projectId, stored.files[0]!)
|
|
else await importYoloFiles(stored.projectId, stored.files)
|
|
await deleteUpload(stored.id)
|
|
}
|