Skip to content

Commit caa6a92

Browse files
johavenjohaven
committed
feat(backend:files): track download, compression and extraction progress
1 parent 7939491 commit caa6a92

20 files changed

Lines changed: 363 additions & 357 deletions

‎backend/src/applications/files/interfaces/download-file.interface.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@ export interface DownloadFileOptions {
1212
publishedPath?: string
1313
getContentInfo?: boolean
1414
maxSize?: number
15+
onProgress?: (bytes: number) => void
1516
signal?: AbortSignal
1617
}

‎backend/src/applications/files/interfaces/file-event.interface.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import type { SpaceEnv } from '../../spaces/models/space-env.model'
2-
import type { FILE_OPERATION } from '../constants/operations'
32
import type { ACTION } from '../../../common/constants'
43
import type { UserModel } from '../../users/models/user.model'
54

65
export interface FileTaskEventEmit {
7-
startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string]
6+
startWatch: [space: SpaceEnv, rPath: string]
87
}
98

109
export interface FileEventType {

‎backend/src/applications/files/interfaces/file-task.interface.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ export interface FileTaskTransferOptions {
99
beforeCommit?: () => Promise<void>
1010
cacheKey: string
1111
crossDevice?: boolean
12-
onTransferStart?: (temporaryPath: string) => void
12+
onTransferStart?: () => void
1313
onProgress?: (bytes: number) => void
1414
overwrite?: boolean
1515
signal: AbortSignal
@@ -20,3 +20,9 @@ export interface FileTaskCopyTaskOptions extends FileTaskTransferOptions {
2020
preserveTimestamps?: boolean
2121
recursive?: boolean
2222
}
23+
24+
export interface FileTaskExtractionEntry {
25+
path: string
26+
isDirectory: boolean
27+
size: number
28+
}

‎backend/src/applications/files/services/files-manager.service.spec.ts‎

Lines changed: 41 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import * as spacesPathUtils from '../../spaces/utils/paths'
1515
import * as spacesPermsUtils from '../../spaces/utils/permissions'
1616
import { DEPTH } from '../../webdav/constants/webdav'
1717
import { ACTION } from '../../../common/constants'
18-
import { FILE_OPERATION } from '../constants/operations'
1918
import { DownloadFileDto } from '../dto/file-operations.dto'
2019
import { FileEvent, FileTaskEvent } from '../events/file-events'
2120
import { FileError, SourceCleanupError } from '../models/file-error'
@@ -43,7 +42,7 @@ vi.mock('node:dns/promises', () => ({
4342

4443
describe(FilesManager.name, () => {
4544
let service: FilesManager
46-
let filesTasksTransfer: { copy: Mock; move: Mock; delete: Mock }
45+
let filesTasksTransfer: { copy: Mock; move: Mock; delete: Mock; createByteProgressHandler: Mock; createExtractionProgressHandler: Mock }
4746
let http: { axiosRef: Mock }
4847
const lookupMock = lookup as Mock
4948
let filesQueries: { moveFiles: Mock; deleteFiles: Mock }
@@ -172,7 +171,13 @@ describe(FilesManager.name, () => {
172171
}
173172
),
174173
move: vi.fn().mockResolvedValue(undefined),
175-
delete: vi.fn().mockResolvedValue(undefined)
174+
delete: vi.fn().mockResolvedValue(undefined),
175+
createByteProgressHandler: vi.fn((space) =>
176+
vi.fn((bytes: number) => {
177+
space.task.props.size = (space.task.props.size || 0) + bytes
178+
})
179+
),
180+
createExtractionProgressHandler: vi.fn().mockReturnValue(vi.fn())
176181
}
177182
filesLockManager = {
178183
create: vi.fn().mockResolvedValue([true, { key: 'lock-1' }]),
@@ -1054,17 +1059,19 @@ describe(FilesManager.name, () => {
10541059

10551060
await service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })
10561061

1057-
expect(space.task.props.totalSize).toBe(55)
1058-
expect(taskEmitSpy).toHaveBeenCalledWith(
1059-
'startWatch',
1060-
space,
1061-
FILE_OPERATION.DOWNLOAD,
1062-
'/tmp/download.txt',
1063-
'/data/users/john/tmp/tasks/.task-1-download.txt'
1064-
)
1062+
expect(space.task.props).toMatchObject({ progress: 1, size: 0, totalSize: 55 })
1063+
expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/download.txt')
10651064
expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-1', '/tmp/download.txt')
10661065
expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
1067-
expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', expect.anything(), 0, 55)
1066+
expect(filesUtils.writeFromStream).toHaveBeenCalledWith(
1067+
'/data/users/john/tmp/tasks/.task-1-download.txt',
1068+
expect.anything(),
1069+
0,
1070+
55,
1071+
undefined,
1072+
expect.any(Function)
1073+
)
1074+
expect(filesTasksTransfer.createByteProgressHandler).toHaveBeenCalledWith(space)
10681075
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', '/tmp/download.txt')
10691076
expect(filesLockManager.create).toHaveBeenCalledWith(
10701077
user,
@@ -1129,6 +1136,9 @@ describe(FilesManager.name, () => {
11291136
describe('compress', () => {
11301137
it('should archive files and emit events', async () => {
11311138
const archive = createArchiveMock()
1139+
archive.finalize.mockImplementationOnce(async () => {
1140+
archive.end('content')
1141+
})
11321142
vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')
11331143
vi.mocked(filesUtils.isPathIsDir).mockImplementation(async (p: string) => p.endsWith('/dir'))
11341144
const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })
@@ -1149,18 +1159,13 @@ describe(FilesManager.name, () => {
11491159
expect(archive.directory).toHaveBeenCalled()
11501160
expect(archive.file).toHaveBeenCalled()
11511161
expect(archive.finalize).toHaveBeenCalled()
1152-
expect(taskEmitSpy).toHaveBeenCalledWith(
1153-
'startWatch',
1154-
space,
1155-
FILE_OPERATION.COMPRESS,
1156-
'/tmp/archive.tar.gz',
1157-
'/data/users/john/tmp/tasks/.task-c-archive.tar.gz'
1158-
)
1162+
expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tar.gz')
11591163
expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tar.gz')
11601164
expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
11611165
expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', {
11621166
highWaterMark: expect.any(Number)
11631167
})
1168+
expect(space.task.props.size).toBe(Buffer.byteLength('content'))
11641169
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', '/tmp/archive.tar.gz')
11651170
})
11661171

@@ -1314,15 +1319,16 @@ describe(FilesManager.name, () => {
13141319

13151320
expect(taskUtils.createTaskTemporaryDir).toHaveBeenCalledWith(user.tasksPath, 'task-d', '/data/users/john/files/archive')
13161321
expect(filesUtils.makeTempDir).not.toHaveBeenCalled()
1317-
expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/tasks/.task-d-archive', undefined, undefined)
1318-
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')
1319-
expect(taskEmitSpy).toHaveBeenCalledWith(
1320-
'startWatch',
1321-
space,
1322-
FILE_OPERATION.DECOMPRESS,
1323-
'/data/users/john/files/archive',
1324-
'/data/users/john/tmp/tasks/.task-d-archive'
1322+
expect(filesTasksTransfer.createExtractionProgressHandler).toHaveBeenCalledWith(space)
1323+
expect(unzipSpy).toHaveBeenCalledWith(
1324+
'/data/users/john/files/archive.zip',
1325+
'/data/users/john/tmp/tasks/.task-d-archive',
1326+
undefined,
1327+
undefined,
1328+
expect.any(Function)
13251329
)
1330+
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')
1331+
expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/data/users/john/files/archive')
13261332
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
13271333
})
13281334

@@ -1340,6 +1346,7 @@ describe(FilesManager.name, () => {
13401346
'/data/users/john/tmp/archive-extract-123',
13411347
true,
13421348
undefined,
1349+
undefined,
13431350
undefined
13441351
)
13451352
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')
@@ -1354,7 +1361,13 @@ describe(FilesManager.name, () => {
13541361

13551362
await service.decompress(user, space)
13561363

1357-
expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123', 60, undefined)
1364+
expect(unzipSpy).toHaveBeenCalledWith(
1365+
'/data/users/john/files/archive.zip',
1366+
'/data/users/john/tmp/archive-extract-123',
1367+
60,
1368+
undefined,
1369+
undefined
1370+
)
13581371
})
13591372

13601373
it('should remove partial extraction and skip add event on failure', async () => {

‎backend/src/applications/files/services/files-manager.service.ts‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'
2323
import { CACHE_LOCK_FILE_TTL } from '../constants/cache'
2424
import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'
2525
import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'
26-
import { FILE_OPERATION } from '../constants/operations'
2726
import { ALL_DOCUMENT_TYPES, DEFAULT_DOCUMENT_TYPES, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'
2827
import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'
2928
import { FileDBProps } from '../interfaces/file-db-props.interface'
@@ -36,6 +35,7 @@ import {
3635
copyFileContent,
3736
copyFiles,
3837
createEmptyFile,
38+
createProgressTransform,
3939
dirName,
4040
dirSize,
4141
fileName,
@@ -486,7 +486,7 @@ export class FilesManager {
486486
// do
487487
if (isMove) {
488488
let sourceCleanupError: SourceCleanupError | undefined
489-
if (useTaskTransfer) {
489+
if (useTaskTransfer && signal) {
490490
sourceCleanupError = await this.filesTasksTransfer.move(user, srcSpace, dstSpace, overwrite, isDir, signal, () => this.delete(user, dstSpace))
491491
} else {
492492
await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
@@ -604,7 +604,12 @@ export class FilesManager {
604604
}
605605

606606
try {
607-
await new DownloadFile(this.http).download(downloadDto, tmpPath, { space, publishedPath: dstPath, signal })
607+
await new DownloadFile(this.http).download(downloadDto, tmpPath, {
608+
space,
609+
publishedPath: dstPath,
610+
signal,
611+
onProgress: isTaskContext ? this.filesTasksTransfer.createByteProgressHandler(space) : undefined
612+
})
608613
signal?.throwIfAborted()
609614
await moveFiles(tmpPath, dstPath)
610615
} catch (e) {
@@ -652,15 +657,21 @@ export class FilesManager {
652657
}
653658
if (isTaskContext) {
654659
space.task!.props.compressInDirectory = dto.compressInDirectory
655-
FileTaskEvent.emit('startWatch', space, FILE_OPERATION.COMPRESS, dstPath, tmpPath)
660+
space.task!.props.size = 0
661+
FileTaskEvent.emit('startWatch', space, dstPath)
656662
}
657663
// do
658664
let aborted = false
659665
let pipePromise: Promise<void> | undefined
660666
let entriesPromise: Promise<void> | undefined
661667
try {
662668
const dstStream = fs.createWriteStream(tmpPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
663-
pipePromise = pipeline(archive, dstStream, { signal }) // handle archive errors + write stream
669+
if (isTaskContext) {
670+
const onProgress = this.filesTasksTransfer.createByteProgressHandler(space)
671+
pipePromise = pipeline(archive, createProgressTransform(onProgress), dstStream, { signal })
672+
} else {
673+
pipePromise = pipeline(archive, dstStream, { signal })
674+
}
664675
entriesPromise = (async () => {
665676
for (const f of dto.files) {
666677
signal?.throwIfAborted()
@@ -723,13 +734,14 @@ export class FilesManager {
723734
}
724735
fileLock = lock
725736
// tasking
726-
if (isTaskContext) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)
737+
const onEntry = isTaskContext ? this.filesTasksTransfer.createExtractionProgressHandler(space) : undefined
738+
if (isTaskContext) FileTaskEvent.emit('startWatch', space, dstPath)
727739
// do
728740
const maxExtractedSize = space.storageQuota === null ? undefined : Math.max(0, space.storageQuota - space.storageUsage)
729741
if (extension === '.zip') {
730-
await extractZip(space.realPath, tmpPath, maxExtractedSize, signal)
742+
await extractZip(space.realPath, tmpPath, maxExtractedSize, signal, onEntry)
731743
} else {
732-
await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION, maxExtractedSize, signal)
744+
await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION, maxExtractedSize, signal, onEntry)
733745
}
734746
signal?.throwIfAborted()
735747
if (await isPathExists(dstPath)) {

0 commit comments

Comments
 (0)