import { api } from './api' import { deleteUpload, prepareUploadStorage, saveUpload, type StoredUpload } from './uploadStore' import { API_BASE_URL, STORAGE_KEYS } from '@/app/constants' import { getStorageItem } from '@/utils/storage' import { importCocoFile, importYoloFiles } from './datasetImportService' export type UploadTaskStatus = 'pending' | 'uploading' | 'done' | 'error' | 'skipped' export interface UploadTask { fileName: string status: UploadTaskStatus progress: number error?: string imageId?: string } export interface BatchUploadResult { done: { fileName: string; imageId: string }[] failed: { fileName: string; error: string }[] skipped: string[] } interface PresignSlot { id: string fileName: string putUrl: string r2Key: string } const CONCURRENCY = 4 const MAX_RETRIES = 2 function putWithProgress( url: string, file: File, contentType: string, onProgress: (ratio: number) => void ): Promise { // fetch has no upload progress; XHR is used for this single call only. return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest() xhr.open('PUT', url) xhr.setRequestHeader('Content-Type', contentType) xhr.upload.onprogress = (e) => { if (e.lengthComputable) onProgress(e.loaded / e.total) } xhr.onload = () => xhr.status >= 200 && xhr.status < 300 ? resolve() : reject(new Error(`Tải lên kho lưu trữ thất bại (${xhr.status})`)) xhr.onerror = () => reject(new Error('Lỗi mạng khi tải lên')) xhr.send(file) }) } async function readDimensions(file: File): Promise<{ width: number; height: number }> { try { const bmp = await createImageBitmap(file) const dims = { width: bmp.width, height: bmp.height } bmp.close() return dims } catch { return { width: 0, height: 0 } } } // Recursively collect files from a drop (files and/or folders) preserving names. export async function collectDroppedFiles(dt: DataTransfer): Promise { const items = Array.from(dt.items).filter((i) => i.kind === 'file') const entries = items .map((i) => (typeof i.webkitGetAsEntry === 'function' ? i.webkitGetAsEntry() : null)) .filter((e): e is FileSystemEntry => e !== null) if (entries.length === 0 || entries.every((e) => e.isFile)) { return Array.from(dt.files) } const out: File[] = [] const walk = async (entry: FileSystemEntry, prefix: string): Promise => { if (entry.isFile) { const blob = await new Promise((resolve, reject) => (entry as FileSystemFileEntry).file(resolve, reject) ) out.push(new File([blob], prefix + blob.name, { type: blob.type })) return } if (entry.isDirectory) { const reader = (entry as FileSystemDirectoryEntry).createReader() for (;;) { const batch: FileSystemEntry[] = await new Promise((resolve, reject) => reader.readEntries(resolve, reject) ) if (batch.length === 0) break for (const child of batch) await walk(child, `${prefix}${entry.name}/`) } } } for (const entry of entries) await walk(entry, '') return out } // ---- Resumable multipart upload for single big files (100MB..GBs) ---- // Parts go straight to R2 in parallel with per-part retry; ETags persist in // localStorage so a dropped network resumes instead of restarting. const MP_PART_SIZE = 8 * 1024 * 1024 const MP_CONCURRENCY = 4 const MP_RETRIES = 2 const MP_RESUME_PREFIX = 'danable:mp:' async function waitForProcessedItem(projectId: string, jobId: string, itemId: string): Promise { const token = getStorageItem(STORAGE_KEYS.ACCESS_TOKEN, null) const response = await fetch(`${API_BASE_URL}/projects/${projectId}/upload-jobs/${jobId}/events`, { headers: token ? { Authorization: `Bearer ${token}` } : undefined, }) if (!response.ok || !response.body) throw new Error(`Không kết nối được SSE (${response.status})`) const reader = response.body.getReader() const decoder = new TextDecoder() let buffer = '' try { for (;;) { const chunk = await reader.read() if (chunk.done) break buffer += decoder.decode(chunk.value, { stream: true }) const events = buffer.split('\n\n') buffer = events.pop() ?? '' for (const event of events) { if (!event.includes('data:')) continue const { data } = await api.get<{ items: Array<{ id: string; status: string; imageId?: string; error?: string }> }>( `/projects/${projectId}/upload-jobs/${jobId}` ) const item = data.items.find((candidate) => candidate.id === itemId) if (item?.status === 'done' && item.imageId) return item.imageId if (item?.status === 'error') throw new Error(item.error || 'Xử lý ảnh thất bại') } } } finally { await reader.cancel() } throw new Error('SSE upload worker đã đóng trước khi xử lý xong') } interface MpResume { key: string uploadId: string etags: Record totalParts: number } function mpFingerprint(file: File): string { return `${MP_RESUME_PREFIX}${file.name}-${file.size}-${file.lastModified}` } function mpStoreId(projectId: string, file: File): string { return `zip:${projectId}:${file.name}:${file.size}:${file.lastModified}` } function putPart( url: string, blob: Blob, onProgress: (loaded: number) => void ): Promise { return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest() xhr.open('PUT', url) xhr.upload.onprogress = (e) => { if (e.lengthComputable) onProgress(e.loaded) } xhr.onload = () => { if (xhr.status >= 200 && xhr.status < 300) { resolve(xhr.getResponseHeader('ETag') ?? '') } else { reject(new Error(`Part thất bại (${xhr.status})`)) } } xhr.onerror = () => reject(new Error('Lỗi mạng khi tải part')) xhr.send(blob) }) } export async function uploadFileMultipart( projectId: string, file: File, onProgress?: (ratio: number, uploadedBytes: number) => void ): Promise<{ r2Key: string }> { const storeId = mpStoreId(projectId, file) await prepareUploadStorage(file.size) await saveUpload({ id: storeId, projectId, jobId: '', files: [file], createdAt: Date.now(), kind: 'zip' }) const totalParts = Math.max(1, Math.ceil(file.size / MP_PART_SIZE)) const saved = (() => { try { const raw = localStorage.getItem(mpFingerprint(file)) return raw ? (JSON.parse(raw) as MpResume) : null } catch { return null } })() let key: string let uploadId: string const etags: Record = { ...(saved?.etags ?? {}) } if (saved && saved.totalParts === totalParts) { // Resume: reuse key + uploadId, skip parts that already have ETags. key = saved.key uploadId = saved.uploadId const { data: adopted } = await api.post<{ jobId: string }>( `/projects/${projectId}/uploads/multipart/adopt`, { fileName: file.name, contentType: file.type || 'application/octet-stream', key, uploadId } ) await saveUpload({ id: storeId, projectId, jobId: adopted.jobId, files: [file], createdAt: Date.now(), kind: 'zip', r2Key: key }) } else { const { data } = await api.post<{ key: string; uploadId: string; jobId?: string }>( `/projects/${projectId}/uploads/multipart`, { fileName: file.name, contentType: file.type || 'application/octet-stream' } ) key = data.key uploadId = data.uploadId await saveUpload({ id: storeId, projectId, jobId: data.jobId ?? '', files: [file], createdAt: Date.now(), kind: 'zip', r2Key: key }) } const persist = () => { try { localStorage.setItem( mpFingerprint(file), JSON.stringify({ key, uploadId, etags, totalParts }) ) } catch { // Private mode: resume simply unavailable. } } persist() const { data: partUrls } = await api.post<{ urls: Record }>( `/projects/${projectId}/uploads/multipart/parts`, { key, uploadId, numbers: Array.from({ length: totalParts }, (_, i) => i + 1) } ) const loaded = new Array(totalParts).fill(0) const report = () => { const sum = loaded.reduce((a, b) => a + b, 0) onProgress?.(sum / Math.max(1, file.size), sum) } let cursor = 0 const numbers = Array.from({ length: totalParts }, (_, i) => i + 1) const worker = async (): Promise => { for (;;) { const n = numbers[cursor++] if (n === undefined) return if (etags[n]) continue const start = (n - 1) * MP_PART_SIZE const blob = file.slice(start, start + MP_PART_SIZE) let done = false for (let attempt = 0; attempt <= MP_RETRIES && !done; attempt++) { try { etags[n] = await putPart(partUrls.urls[n]!, blob, (ld) => { loaded[n - 1] = ld report() }) loaded[n - 1] = blob.size report() persist() done = true } catch (err) { if (attempt === MP_RETRIES) throw err } } } } await Promise.all(Array.from({ length: Math.min(MP_CONCURRENCY, totalParts) }, worker)) await api.post(`/projects/${projectId}/uploads/multipart/complete`, { key, uploadId, parts: Object.entries(etags).map(([num, ETag]) => ({ ETag, PartNumber: Number(num) })), }) try { localStorage.removeItem(mpFingerprint(file)) } catch { // ignore } onProgress?.(1, file.size) return { r2Key: key } } export async function uploadFilesBatch( projectId: string, files: File[], onUpdate?: (tasks: UploadTask[]) => void ): Promise { const seen = new Set() const tasks: UploadTask[] = files.map((f) => { if (seen.has(f.name)) { return { fileName: f.name, status: 'skipped', progress: 0 } } seen.add(f.name) return { fileName: f.name, status: 'pending', progress: 0 } }) const emit = () => onUpdate?.(tasks.map((t) => ({ ...t }))) emit() const pending = files.filter((f) => tasks.find((t) => t.fileName === f.name)?.status === 'pending') if (pending.length === 0) { return { done: [], failed: [], skipped: tasks.map((t) => t.fileName) } } await prepareUploadStorage(pending.reduce((sum, file) => sum + file.size, 0)) const { data: created } = await api.post<{ job: { id: string } items: PresignSlot[] }>(`/projects/${projectId}/upload-jobs`, { kind: 'images', files: pending.map((file) => ({ fileName: file.name, contentType: file.type || 'image/jpeg', size: file.size, })), }) const jobId = created.job.id await saveUpload({ id: jobId, projectId, jobId, files: pending, createdAt: Date.now(), }) const slots = new Map() created.items.forEach((s) => slots.set(s.fileName, s)) const result: BatchUploadResult = { done: [], failed: [], skipped: [] } tasks.forEach((t) => { if (t.status === 'skipped') result.skipped.push(t.fileName) }) let cursor = 0 const queue = pending.slice() const worker = async (): Promise => { for (;;) { const file = queue[cursor++] if (!file) return const task = tasks.find((t) => t.fileName === file.name)! const slot = slots.get(file.name) if (!slot) { task.status = 'error' task.error = 'Không lấy được URL tải lên' result.failed.push({ fileName: file.name, error: task.error }) emit() continue } task.status = 'uploading' emit() let ok = false for (let attempt = 0; attempt <= MAX_RETRIES && !ok; attempt++) { try { const dims = await readDimensions(file) await putWithProgress(slot.putUrl, file, file.type || 'image/jpeg', (r) => { task.progress = r emit() }) await api.post<{ processedFiles: number }>( `/projects/${projectId}/upload-jobs/${jobId}/items/${slot.id}/complete`, { width: dims.width, height: dims.height } ) const imageId = await waitForProcessedItem(projectId, jobId, slot.id) task.status = 'done' task.progress = 1 task.imageId = imageId result.done.push({ fileName: file.name, imageId }) ok = true } catch (err) { task.error = err instanceof Error ? err.message : 'Tải lên thất bại' if (attempt === MAX_RETRIES) { task.status = 'error' result.failed.push({ fileName: file.name, error: task.error }) } } finally { emit() } } } } await Promise.all(Array.from({ length: Math.min(CONCURRENCY, queue.length) }, worker)) if (result.failed.length === 0) await deleteUpload(jobId) return result } export async function resumeStoredUpload( stored: StoredUpload, onProgress?: (done: number, total: number) => void ): Promise { const { data: job } = await api.get<{ status: string totalFiles: number processedFiles: number items: Array<{ id: string; fileName: string; status: string; putUrl?: string }> }>(`/projects/${stored.projectId}/upload-jobs/${stored.jobId}`) if (job.status === 'done') { await deleteUpload(stored.id) return } const byName = new Map(stored.files.map((file) => [file.name, file])) let done = job.processedFiles onProgress?.(done, job.totalFiles) for (const item of job.items) { if (item.status === 'done') continue const file = byName.get(item.fileName) if (!file || !item.putUrl) throw new Error(`Không tìm thấy file để resume: ${item.fileName}`) const dims = await readDimensions(file) await putWithProgress(item.putUrl, file, file.type || 'image/jpeg', () => undefined) await api.post( `/projects/${stored.projectId}/upload-jobs/${stored.jobId}/items/${item.id}/complete`, { width: dims.width, height: dims.height } ) await waitForProcessedItem(stored.projectId, stored.jobId, item.id) done += 1 onProgress?.(done, job.totalFiles) } await deleteUpload(stored.id) } async function resumeImportJob(projectId: string, jobId: string, onProgress?: (done: number, total: number) => void) { const token = getStorageItem(STORAGE_KEYS.ACCESS_TOKEN, null) const response = await fetch(`${API_BASE_URL}/projects/${projectId}/imports/${jobId}/events`, { headers: token ? { Authorization: `Bearer ${token}` } : undefined, }) if (!response.ok || !response.body) throw new Error(`Không kết nối được SSE import (${response.status})`) const reader = response.body.getReader() const decoder = new TextDecoder() let buffer = '' try { for (;;) { const chunk = await reader.read() if (chunk.done) break buffer += decoder.decode(chunk.value, { stream: true }) const events = buffer.split('\n\n') buffer = events.pop() ?? '' for (const event of events) { const line = event.split('\n').find((value) => value.startsWith('data:')) if (!line) continue const data = JSON.parse(line.slice(5)) as { status: string; done_files: number; total_files: number; error?: string } onProgress?.(data.done_files, data.total_files) if (data.status === 'done') return if (data.status === 'error') throw new Error(data.error || 'Import ZIP thất bại') } } } finally { await reader.cancel() } throw new Error('SSE import đã đóng trước khi hoàn tất') } export async function resumeStoredZip(stored: StoredUpload, onProgress?: (message: string) => void) { const file = stored.files[0] if (!file) throw new Error('Không còn file ZIP để resume') if (stored.kind === 'zip-import' && stored.jobId) { await resumeImportJob(stored.projectId, stored.jobId, (done, total) => onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`)) await deleteUpload(stored.id) return } const { r2Key } = await uploadFileMultipart(stored.projectId, file, (ratio) => onProgress?.(`Đang resume ZIP ${Math.round(ratio * 100)}%`)) const { data: job } = await api.post<{ id: string }>( `/projects/${stored.projectId}/imports/r2-zip`, { r2_key: r2Key }) await saveUpload({ ...stored, kind: 'zip-import', jobId: job.id, r2Key }) await resumeImportJob(stored.projectId, job.id, (done, total) => onProgress?.(`Đang import ZIP ${done}/${total} ảnh…`)) await deleteUpload(stored.id) } export async function resumeStoredAnnotationImport(stored: StoredUpload) { if (stored.kind === 'coco') await importCocoFile(stored.projectId, stored.files[0]!) else await importYoloFiles(stored.projectId, stored.files) await deleteUpload(stored.id) }