From 3906a523fd5957198ce4ad6cece5f18c2f99fd1b Mon Sep 17 00:00:00 2001 From: yoyuzh Date: Thu, 9 Apr 2026 16:00:34 +0800 Subject: [PATCH] refactor(files): reorganize backend package layout --- .../java/com/yoyuzh/admin/AdminService.java | 14 +- .../admin/AdminStoragePolicyResponse.java | 6 +- .../api/v2/files/FileEventsV2Controller.java | 2 +- .../api/v2/files/FileSearchV2Controller.java | 6 +- .../v2/files/PreparedUploadV2Response.java | 12 + .../v2/files/UploadSessionV2Controller.java | 25 +- .../api/v2/files/UploadSessionV2Response.java | 1 + .../api/v2/shares/ShareV2Controller.java | 11 +- .../yoyuzh/api/v2/shares/ShareV2Response.java | 2 +- .../api/v2/tasks/BackgroundTaskResponse.java | 4 +- .../v2/tasks/BackgroundTaskV2Controller.java | 13 +- .../java/com/yoyuzh/auth/AuthService.java | 4 +- .../auth/DevBootstrapDataInitializer.java | 4 +- .../yoyuzh/files/BackgroundTaskHandler.java | 8 - .../files/BackgroundTaskRepository.java | 35 - .../yoyuzh/files/BackgroundTaskService.java | 314 -------- .../yoyuzh/files/BackgroundTaskWorker.java | 61 -- .../files/{ => core}/CopyFileRequest.java | 2 +- .../files/{ => core}/DownloadUrlResponse.java | 2 +- .../com/yoyuzh/files/{ => core}/FileBlob.java | 2 +- .../{ => core}/FileBlobBackfillService.java | 2 +- .../files/{ => core}/FileBlobRepository.java | 2 +- .../files/{ => core}/FileController.java | 8 +- .../yoyuzh/files/{ => core}/FileEntity.java | 2 +- .../{ => core}/FileEntityBackfillService.java | 3 +- .../{ => core}/FileEntityRepository.java | 2 +- .../files/{ => core}/FileEntityType.java | 2 +- .../{ => core}/FileMetadataResponse.java | 2 +- .../yoyuzh/files/{ => core}/FileService.java | 446 +++++++++++- .../yoyuzh/files/{ => core}/MkdirRequest.java | 2 +- .../files/{ => core}/MoveFileRequest.java | 2 +- .../{ => core}/RecycleBinItemResponse.java | 2 +- .../files/{ => core}/RenameFileRequest.java | 2 +- .../yoyuzh/files/{ => core}/StoredFile.java | 2 +- .../files/{ => core}/StoredFileEntity.java | 2 +- .../StoredFileEntityRepository.java | 2 +- .../{ => core}/StoredFileRepository.java | 2 +- .../yoyuzh/files/{ => events}/FileEvent.java | 2 +- .../{ => events}/FileEventRepository.java | 2 +- .../files/{ => events}/FileEventService.java | 2 +- .../files/{ => events}/FileEventType.java | 2 +- .../files/{ => policy}/StoragePolicy.java | 2 +- .../StoragePolicyCapabilities.java | 2 +- .../StoragePolicyCredentialMode.java | 2 +- .../{ => policy}/StoragePolicyRepository.java | 2 +- .../{ => policy}/StoragePolicyService.java | 4 +- .../files/{ => policy}/StoragePolicyType.java | 2 +- .../files/{ => search}/FileMetadata.java | 3 +- .../{ => search}/FileMetadataRepository.java | 2 +- .../files/{ => search}/FileSearchQuery.java | 2 +- .../files/{ => search}/FileSearchService.java | 5 +- .../CreateFileShareLinkResponse.java | 2 +- .../{ => share}/FileShareDetailsResponse.java | 2 +- .../files/{ => share}/FileShareLink.java | 3 +- .../{ => share}/FileShareLinkRepository.java | 2 +- .../{ => share}/ImportSharedFileRequest.java | 2 +- .../files/{ => share}/ShareV2Service.java | 30 +- .../files/storage/FileContentStorage.java | 20 + .../files/storage/MultipartCompletedPart.java | 7 + .../files/storage/S3FileContentStorage.java | 100 +++ .../tasks/ArchiveBackgroundTaskHandler.java | 164 +++++ .../files/{ => tasks}/BackgroundTask.java | 75 +- .../tasks/BackgroundTaskFailureCategory.java | 19 + .../files/tasks/BackgroundTaskHandler.java | 12 + .../BackgroundTaskHandlerResult.java | 2 +- .../BackgroundTaskLeaseLostException.java | 8 + .../tasks/BackgroundTaskProgressReporter.java | 9 + .../files/tasks/BackgroundTaskRepository.java | 103 +++ .../files/tasks/BackgroundTaskService.java | 683 ++++++++++++++++++ .../tasks/BackgroundTaskStartupRecovery.java | 23 + .../{ => tasks}/BackgroundTaskStatus.java | 2 +- .../files/{ => tasks}/BackgroundTaskType.java | 2 +- .../files/tasks/BackgroundTaskWorker.java | 167 +++++ .../tasks/ExtractBackgroundTaskHandler.java | 356 +++++++++ .../MediaMetadataBackgroundTaskHandler.java | 16 +- .../NoopBackgroundTaskHandler.java | 7 +- .../{ => upload}/CompleteUploadRequest.java | 2 +- .../{ => upload}/InitiateUploadRequest.java | 2 +- .../{ => upload}/InitiateUploadResponse.java | 2 +- .../files/{ => upload}/UploadSession.java | 13 +- .../UploadSessionCreateCommand.java | 2 +- .../UploadSessionPartCommand.java | 2 +- .../{ => upload}/UploadSessionRepository.java | 2 +- .../{ => upload}/UploadSessionService.java | 82 ++- .../{ => upload}/UploadSessionStatus.java | 2 +- .../yoyuzh/transfer/TransferController.java | 4 +- .../com/yoyuzh/transfer/TransferService.java | 4 +- .../admin/AdminControllerIntegrationTest.java | 8 +- .../com/yoyuzh/admin/AdminServiceTest.java | 12 +- .../v2/files/FileSearchV2ControllerTest.java | 6 +- .../files/UploadSessionV2ControllerTest.java | 35 +- .../ShareV2ControllerIntegrationTest.java | 128 +++- ...groundTaskV2ControllerIntegrationTest.java | 453 +++++++++++- .../java/com/yoyuzh/auth/AuthServiceTest.java | 4 +- .../auth/AuthSingleDeviceIntegrationTest.java | 2 +- .../auth/DevBootstrapDataInitializerTest.java | 4 +- .../files/BackgroundTaskServiceTest.java | 246 ------- .../files/BackgroundTaskWorkerTest.java | 83 --- .../FileBlobBackfillServiceTest.java | 2 +- .../FileEntityBackfillServiceTest.java | 6 +- .../{ => core}/FileServiceEdgeCaseTest.java | 4 +- .../files/{ => core}/FileServiceTest.java | 182 ++++- .../FileShareControllerIntegrationTest.java | 3 +- .../RecycleBinControllerIntegrationTest.java | 2 +- .../FileEventPersistenceIntegrationTest.java | 11 +- .../StoragePolicyServiceTest.java | 4 +- .../{ => search}/FileSearchServiceTest.java | 4 +- .../storage/S3FileContentStorageTest.java | 73 ++ .../BackgroundTaskArchiveHandlerTest.java | 234 ++++++ .../tasks/BackgroundTaskServiceTest.java | 614 ++++++++++++++++ .../files/tasks/BackgroundTaskWorkerTest.java | 139 ++++ .../ExtractBackgroundTaskHandlerTest.java | 193 +++++ ...ediaMetadataBackgroundTaskHandlerTest.java | 13 +- .../UploadSessionServiceTest.java | 70 +- docs/agents/unfinished-work.md | 57 +- docs/api-reference.md | 56 +- docs/architecture.md | 49 +- memory.md | 23 +- 118 files changed, 4722 insertions(+), 978 deletions(-) create mode 100644 backend/src/main/java/com/yoyuzh/api/v2/files/PreparedUploadV2Response.java delete mode 100644 backend/src/main/java/com/yoyuzh/files/BackgroundTaskHandler.java delete mode 100644 backend/src/main/java/com/yoyuzh/files/BackgroundTaskRepository.java delete mode 100644 backend/src/main/java/com/yoyuzh/files/BackgroundTaskService.java delete mode 100644 backend/src/main/java/com/yoyuzh/files/BackgroundTaskWorker.java rename backend/src/main/java/com/yoyuzh/files/{ => core}/CopyFileRequest.java (84%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/DownloadUrlResponse.java (62%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileBlob.java (98%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileBlobBackfillService.java (98%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileBlobRepository.java (93%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileController.java (96%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileEntity.java (99%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileEntityBackfillService.java (96%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileEntityRepository.java (89%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileEntityType.java (76%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileMetadataResponse.java (88%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/FileService.java (74%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/MkdirRequest.java (77%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/MoveFileRequest.java (84%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/RecycleBinItemResponse.java (90%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/RenameFileRequest.java (84%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/StoredFile.java (99%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/StoredFileEntity.java (98%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/StoredFileEntityRepository.java (83%) rename backend/src/main/java/com/yoyuzh/files/{ => core}/StoredFileRepository.java (99%) rename backend/src/main/java/com/yoyuzh/files/{ => events}/FileEvent.java (98%) rename backend/src/main/java/com/yoyuzh/files/{ => events}/FileEventRepository.java (81%) rename backend/src/main/java/com/yoyuzh/files/{ => events}/FileEventService.java (99%) rename backend/src/main/java/com/yoyuzh/files/{ => events}/FileEventType.java (76%) rename backend/src/main/java/com/yoyuzh/files/{ => policy}/StoragePolicy.java (99%) rename backend/src/main/java/com/yoyuzh/files/{ => policy}/StoragePolicyCapabilities.java (91%) rename backend/src/main/java/com/yoyuzh/files/{ => policy}/StoragePolicyCredentialMode.java (72%) rename backend/src/main/java/com/yoyuzh/files/{ => policy}/StoragePolicyRepository.java (88%) rename backend/src/main/java/com/yoyuzh/files/{ => policy}/StoragePolicyService.java (98%) rename backend/src/main/java/com/yoyuzh/files/{ => policy}/StoragePolicyType.java (65%) rename backend/src/main/java/com/yoyuzh/files/{ => search}/FileMetadata.java (97%) rename backend/src/main/java/com/yoyuzh/files/{ => search}/FileMetadataRepository.java (88%) rename backend/src/main/java/com/yoyuzh/files/{ => search}/FileSearchQuery.java (91%) rename backend/src/main/java/com/yoyuzh/files/{ => search}/FileSearchService.java (95%) rename backend/src/main/java/com/yoyuzh/files/{ => share}/CreateFileShareLinkResponse.java (86%) rename backend/src/main/java/com/yoyuzh/files/{ => share}/FileShareDetailsResponse.java (89%) rename backend/src/main/java/com/yoyuzh/files/{ => share}/FileShareLink.java (98%) rename backend/src/main/java/com/yoyuzh/files/{ => share}/FileShareLinkRepository.java (96%) rename backend/src/main/java/com/yoyuzh/files/{ => share}/ImportSharedFileRequest.java (78%) rename backend/src/main/java/com/yoyuzh/files/{ => share}/ShareV2Service.java (87%) create mode 100644 backend/src/main/java/com/yoyuzh/files/storage/MultipartCompletedPart.java create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/ArchiveBackgroundTaskHandler.java rename backend/src/main/java/com/yoyuzh/files/{ => tasks}/BackgroundTask.java (71%) create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskFailureCategory.java create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskHandler.java rename backend/src/main/java/com/yoyuzh/files/{ => tasks}/BackgroundTaskHandlerResult.java (87%) create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskLeaseLostException.java create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskProgressReporter.java create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskRepository.java create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskService.java create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskStartupRecovery.java rename backend/src/main/java/com/yoyuzh/files/{ => tasks}/BackgroundTaskStatus.java (76%) rename backend/src/main/java/com/yoyuzh/files/{ => tasks}/BackgroundTaskType.java (81%) create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskWorker.java create mode 100644 backend/src/main/java/com/yoyuzh/files/tasks/ExtractBackgroundTaskHandler.java rename backend/src/main/java/com/yoyuzh/files/{ => tasks}/MediaMetadataBackgroundTaskHandler.java (90%) rename backend/src/main/java/com/yoyuzh/files/{ => tasks}/NoopBackgroundTaskHandler.java (85%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/CompleteUploadRequest.java (89%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/InitiateUploadRequest.java (88%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/InitiateUploadResponse.java (86%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/UploadSession.java (94%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/UploadSessionCreateCommand.java (81%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/UploadSessionPartCommand.java (72%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/UploadSessionRepository.java (93%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/UploadSessionService.java (76%) rename backend/src/main/java/com/yoyuzh/files/{ => upload}/UploadSessionStatus.java (80%) delete mode 100644 backend/src/test/java/com/yoyuzh/files/BackgroundTaskServiceTest.java delete mode 100644 backend/src/test/java/com/yoyuzh/files/BackgroundTaskWorkerTest.java rename backend/src/test/java/com/yoyuzh/files/{ => core}/FileBlobBackfillServiceTest.java (99%) rename backend/src/test/java/com/yoyuzh/files/{ => core}/FileEntityBackfillServiceTest.java (95%) rename backend/src/test/java/com/yoyuzh/files/{ => core}/FileServiceEdgeCaseTest.java (98%) rename backend/src/test/java/com/yoyuzh/files/{ => core}/FileServiceTest.java (82%) rename backend/src/test/java/com/yoyuzh/files/{ => core}/FileShareControllerIntegrationTest.java (99%) rename backend/src/test/java/com/yoyuzh/files/{ => core}/RecycleBinControllerIntegrationTest.java (99%) rename backend/src/test/java/com/yoyuzh/files/{ => events}/FileEventPersistenceIntegrationTest.java (88%) rename backend/src/test/java/com/yoyuzh/files/{ => policy}/StoragePolicyServiceTest.java (97%) rename backend/src/test/java/com/yoyuzh/files/{ => search}/FileSearchServiceTest.java (97%) create mode 100644 backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskArchiveHandlerTest.java create mode 100644 backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskServiceTest.java create mode 100644 backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskWorkerTest.java create mode 100644 backend/src/test/java/com/yoyuzh/files/tasks/ExtractBackgroundTaskHandlerTest.java rename backend/src/test/java/com/yoyuzh/files/{ => tasks}/MediaMetadataBackgroundTaskHandlerTest.java (95%) rename backend/src/test/java/com/yoyuzh/files/{ => upload}/UploadSessionServiceTest.java (77%) diff --git a/backend/src/main/java/com/yoyuzh/admin/AdminService.java b/backend/src/main/java/com/yoyuzh/admin/AdminService.java index f185066..331d182 100644 --- a/backend/src/main/java/com/yoyuzh/admin/AdminService.java +++ b/backend/src/main/java/com/yoyuzh/admin/AdminService.java @@ -9,13 +9,13 @@ import com.yoyuzh.auth.RefreshTokenService; import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.PageResponse; -import com.yoyuzh.files.FileBlobRepository; -import com.yoyuzh.files.FileService; -import com.yoyuzh.files.StoredFile; -import com.yoyuzh.files.StoredFileRepository; -import com.yoyuzh.files.StoragePolicy; -import com.yoyuzh.files.StoragePolicyRepository; -import com.yoyuzh.files.StoragePolicyService; +import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.policy.StoragePolicy; +import com.yoyuzh.files.policy.StoragePolicyRepository; +import com.yoyuzh.files.policy.StoragePolicyService; import com.yoyuzh.transfer.OfflineTransferSessionRepository; import lombok.RequiredArgsConstructor; import org.springframework.data.domain.Page; diff --git a/backend/src/main/java/com/yoyuzh/admin/AdminStoragePolicyResponse.java b/backend/src/main/java/com/yoyuzh/admin/AdminStoragePolicyResponse.java index 3e042c2..2653d1a 100644 --- a/backend/src/main/java/com/yoyuzh/admin/AdminStoragePolicyResponse.java +++ b/backend/src/main/java/com/yoyuzh/admin/AdminStoragePolicyResponse.java @@ -1,8 +1,8 @@ package com.yoyuzh.admin; -import com.yoyuzh.files.StoragePolicyCapabilities; -import com.yoyuzh.files.StoragePolicyCredentialMode; -import com.yoyuzh.files.StoragePolicyType; +import com.yoyuzh.files.policy.StoragePolicyCapabilities; +import com.yoyuzh.files.policy.StoragePolicyCredentialMode; +import com.yoyuzh.files.policy.StoragePolicyType; import java.time.LocalDateTime; diff --git a/backend/src/main/java/com/yoyuzh/api/v2/files/FileEventsV2Controller.java b/backend/src/main/java/com/yoyuzh/api/v2/files/FileEventsV2Controller.java index d2e92cb..75f4b74 100644 --- a/backend/src/main/java/com/yoyuzh/api/v2/files/FileEventsV2Controller.java +++ b/backend/src/main/java/com/yoyuzh/api/v2/files/FileEventsV2Controller.java @@ -2,7 +2,7 @@ package com.yoyuzh.api.v2.files; import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.User; -import com.yoyuzh.files.FileEventService; +import com.yoyuzh.files.events.FileEventService; import lombok.RequiredArgsConstructor; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.core.userdetails.UserDetails; diff --git a/backend/src/main/java/com/yoyuzh/api/v2/files/FileSearchV2Controller.java b/backend/src/main/java/com/yoyuzh/api/v2/files/FileSearchV2Controller.java index fd52667..7c4abfd 100644 --- a/backend/src/main/java/com/yoyuzh/api/v2/files/FileSearchV2Controller.java +++ b/backend/src/main/java/com/yoyuzh/api/v2/files/FileSearchV2Controller.java @@ -6,9 +6,9 @@ import com.yoyuzh.api.v2.ApiV2Response; import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.User; import com.yoyuzh.common.PageResponse; -import com.yoyuzh.files.FileMetadataResponse; -import com.yoyuzh.files.FileSearchQuery; -import com.yoyuzh.files.FileSearchService; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.search.FileSearchQuery; +import com.yoyuzh.files.search.FileSearchService; import lombok.RequiredArgsConstructor; import org.springframework.format.annotation.DateTimeFormat; import org.springframework.security.core.annotation.AuthenticationPrincipal; diff --git a/backend/src/main/java/com/yoyuzh/api/v2/files/PreparedUploadV2Response.java b/backend/src/main/java/com/yoyuzh/api/v2/files/PreparedUploadV2Response.java new file mode 100644 index 0000000..19ddbe8 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/api/v2/files/PreparedUploadV2Response.java @@ -0,0 +1,12 @@ +package com.yoyuzh.api.v2.files; + +import java.util.Map; + +public record PreparedUploadV2Response( + boolean direct, + String uploadUrl, + String method, + Map headers, + String storageName +) { +} diff --git a/backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Controller.java b/backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Controller.java index b7c4898..5a8fbe7 100644 --- a/backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Controller.java +++ b/backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Controller.java @@ -3,10 +3,11 @@ package com.yoyuzh.api.v2.files; import com.yoyuzh.api.v2.ApiV2Response; import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.User; -import com.yoyuzh.files.UploadSession; -import com.yoyuzh.files.UploadSessionCreateCommand; -import com.yoyuzh.files.UploadSessionPartCommand; -import com.yoyuzh.files.UploadSessionService; +import com.yoyuzh.files.upload.UploadSession; +import com.yoyuzh.files.upload.UploadSessionCreateCommand; +import com.yoyuzh.files.upload.UploadSessionPartCommand; +import com.yoyuzh.files.upload.UploadSessionService; +import com.yoyuzh.files.storage.PreparedUpload; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import org.springframework.security.core.annotation.AuthenticationPrincipal; @@ -77,10 +78,26 @@ public class UploadSessionV2Controller { return ApiV2Response.success(toResponse(session)); } + @GetMapping("/{sessionId}/parts/{partIndex}/prepare") + public ApiV2Response preparePartUpload(@AuthenticationPrincipal UserDetails userDetails, + @PathVariable String sessionId, + @PathVariable int partIndex) { + User user = userDetailsService.loadDomainUser(userDetails.getUsername()); + PreparedUpload preparedUpload = uploadSessionService.prepareOwnedPartUpload(user, sessionId, partIndex); + return ApiV2Response.success(new PreparedUploadV2Response( + preparedUpload.direct(), + preparedUpload.uploadUrl(), + preparedUpload.method(), + preparedUpload.headers(), + preparedUpload.storageName() + )); + } + private UploadSessionV2Response toResponse(UploadSession session) { return new UploadSessionV2Response( session.getSessionId(), session.getObjectKey(), + session.getMultipartUploadId() != null, session.getTargetPath(), session.getFilename(), session.getContentType(), diff --git a/backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Response.java b/backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Response.java index 1200175..bb4aea2 100644 --- a/backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Response.java +++ b/backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Response.java @@ -5,6 +5,7 @@ import java.time.LocalDateTime; public record UploadSessionV2Response( String sessionId, String objectKey, + boolean multipartUpload, String path, String filename, String contentType, diff --git a/backend/src/main/java/com/yoyuzh/api/v2/shares/ShareV2Controller.java b/backend/src/main/java/com/yoyuzh/api/v2/shares/ShareV2Controller.java index 05c8dd8..7069f9e 100644 --- a/backend/src/main/java/com/yoyuzh/api/v2/shares/ShareV2Controller.java +++ b/backend/src/main/java/com/yoyuzh/api/v2/shares/ShareV2Controller.java @@ -4,14 +4,15 @@ import com.yoyuzh.api.v2.ApiV2Response; import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.User; import com.yoyuzh.common.PageResponse; -import com.yoyuzh.files.FileMetadataResponse; -import com.yoyuzh.files.ShareV2Service; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.share.ShareV2Service; import jakarta.validation.Valid; import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import lombok.RequiredArgsConstructor; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; +import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.core.userdetails.UserDetails; import org.springframework.web.bind.annotation.DeleteMapping; @@ -43,6 +44,12 @@ public class ShareV2Controller { return ApiV2Response.success(shareV2Service.getShare(token)); } + @GetMapping(value = "/{token}", params = "download") + public ResponseEntity downloadShare(@PathVariable String token, + @RequestParam(required = false) String password) { + return shareV2Service.downloadSharedFile(token, password); + } + @PostMapping("/{token}/verify-password") public ApiV2Response verifyPassword(@PathVariable String token, @Valid @RequestBody VerifySharePasswordV2Request request) { diff --git a/backend/src/main/java/com/yoyuzh/api/v2/shares/ShareV2Response.java b/backend/src/main/java/com/yoyuzh/api/v2/shares/ShareV2Response.java index 2e20de6..37039ad 100644 --- a/backend/src/main/java/com/yoyuzh/api/v2/shares/ShareV2Response.java +++ b/backend/src/main/java/com/yoyuzh/api/v2/shares/ShareV2Response.java @@ -1,6 +1,6 @@ package com.yoyuzh.api.v2.shares; -import com.yoyuzh.files.FileMetadataResponse; +import com.yoyuzh.files.core.FileMetadataResponse; import java.time.LocalDateTime; diff --git a/backend/src/main/java/com/yoyuzh/api/v2/tasks/BackgroundTaskResponse.java b/backend/src/main/java/com/yoyuzh/api/v2/tasks/BackgroundTaskResponse.java index 44804c8..61927b0 100644 --- a/backend/src/main/java/com/yoyuzh/api/v2/tasks/BackgroundTaskResponse.java +++ b/backend/src/main/java/com/yoyuzh/api/v2/tasks/BackgroundTaskResponse.java @@ -1,7 +1,7 @@ package com.yoyuzh.api.v2.tasks; -import com.yoyuzh.files.BackgroundTaskStatus; -import com.yoyuzh.files.BackgroundTaskType; +import com.yoyuzh.files.tasks.BackgroundTaskStatus; +import com.yoyuzh.files.tasks.BackgroundTaskType; import java.time.LocalDateTime; diff --git a/backend/src/main/java/com/yoyuzh/api/v2/tasks/BackgroundTaskV2Controller.java b/backend/src/main/java/com/yoyuzh/api/v2/tasks/BackgroundTaskV2Controller.java index fd8b67f..402f907 100644 --- a/backend/src/main/java/com/yoyuzh/api/v2/tasks/BackgroundTaskV2Controller.java +++ b/backend/src/main/java/com/yoyuzh/api/v2/tasks/BackgroundTaskV2Controller.java @@ -4,9 +4,9 @@ import com.yoyuzh.api.v2.ApiV2Response; import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.User; import com.yoyuzh.common.PageResponse; -import com.yoyuzh.files.BackgroundTask; -import com.yoyuzh.files.BackgroundTaskService; -import com.yoyuzh.files.BackgroundTaskType; +import com.yoyuzh.files.tasks.BackgroundTask; +import com.yoyuzh.files.tasks.BackgroundTaskService; +import com.yoyuzh.files.tasks.BackgroundTaskType; import jakarta.validation.Valid; import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; @@ -63,6 +63,13 @@ public class BackgroundTaskV2Controller { return ApiV2Response.success(toResponse(backgroundTaskService.cancelOwnedTask(user, id))); } + @PostMapping("/{id}/retry") + public ApiV2Response retry(@AuthenticationPrincipal UserDetails userDetails, + @PathVariable Long id) { + User user = userDetailsService.loadDomainUser(userDetails.getUsername()); + return ApiV2Response.success(toResponse(backgroundTaskService.retryOwnedTask(user, id))); + } + @PostMapping("/archive") public ApiV2Response createArchiveTask(@AuthenticationPrincipal UserDetails userDetails, @Valid @RequestBody CreateBackgroundTaskRequest request) { diff --git a/backend/src/main/java/com/yoyuzh/auth/AuthService.java b/backend/src/main/java/com/yoyuzh/auth/AuthService.java index 5c62f14..fa3ce87 100644 --- a/backend/src/main/java/com/yoyuzh/auth/AuthService.java +++ b/backend/src/main/java/com/yoyuzh/auth/AuthService.java @@ -9,8 +9,8 @@ import com.yoyuzh.auth.dto.UpdateUserProfileRequest; import com.yoyuzh.auth.dto.UserProfileResponse; import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.ErrorCode; -import com.yoyuzh.files.FileService; -import com.yoyuzh.files.InitiateUploadResponse; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.upload.InitiateUploadResponse; import com.yoyuzh.files.storage.FileContentStorage; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; diff --git a/backend/src/main/java/com/yoyuzh/auth/DevBootstrapDataInitializer.java b/backend/src/main/java/com/yoyuzh/auth/DevBootstrapDataInitializer.java index 88b25a2..9640f92 100644 --- a/backend/src/main/java/com/yoyuzh/auth/DevBootstrapDataInitializer.java +++ b/backend/src/main/java/com/yoyuzh/auth/DevBootstrapDataInitializer.java @@ -1,7 +1,7 @@ package com.yoyuzh.auth; -import com.yoyuzh.files.FileService; -import com.yoyuzh.files.StoredFileRepository; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFileRepository; import lombok.RequiredArgsConstructor; import org.springframework.boot.CommandLineRunner; import org.springframework.context.annotation.Profile; diff --git a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskHandler.java b/backend/src/main/java/com/yoyuzh/files/BackgroundTaskHandler.java deleted file mode 100644 index 6723e70..0000000 --- a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskHandler.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.yoyuzh.files; - -public interface BackgroundTaskHandler { - - boolean supports(BackgroundTaskType type); - - BackgroundTaskHandlerResult handle(BackgroundTask task); -} diff --git a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskRepository.java b/backend/src/main/java/com/yoyuzh/files/BackgroundTaskRepository.java deleted file mode 100644 index 23c2302..0000000 --- a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskRepository.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.yoyuzh.files; - -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; - -import java.time.LocalDateTime; -import java.util.List; -import java.util.Optional; - -public interface BackgroundTaskRepository extends JpaRepository { - - Page findByUserIdOrderByCreatedAtDesc(Long userId, Pageable pageable); - - Optional findByIdAndUserId(Long id, Long userId); - - List findByStatusOrderByCreatedAtAsc(BackgroundTaskStatus status, Pageable pageable); - - @Modifying - @Query(""" - update BackgroundTask task - set task.status = :runningStatus, - task.errorMessage = null, - task.updatedAt = :updatedAt - where task.id = :id - and task.status = :queuedStatus - """) - int claimQueuedTask(@Param("id") Long id, - @Param("queuedStatus") BackgroundTaskStatus queuedStatus, - @Param("runningStatus") BackgroundTaskStatus runningStatus, - @Param("updatedAt") LocalDateTime updatedAt); -} diff --git a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskService.java b/backend/src/main/java/com/yoyuzh/files/BackgroundTaskService.java deleted file mode 100644 index 538ca5c..0000000 --- a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskService.java +++ /dev/null @@ -1,314 +0,0 @@ -package com.yoyuzh.files; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yoyuzh.api.v2.ApiV2ErrorCode; -import com.yoyuzh.api.v2.ApiV2Exception; -import com.yoyuzh.auth.User; -import jakarta.transaction.Transactional; -import lombok.RequiredArgsConstructor; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; - -import java.time.LocalDateTime; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; - -@Service -@RequiredArgsConstructor -public class BackgroundTaskService { - - private static final List ARCHIVE_EXTENSIONS = List.of( - ".zip", ".jar", ".war", ".7z", ".rar", ".tar", ".gz", ".tgz", ".bz2", ".xz" - ); - private static final List MEDIA_EXTENSIONS = List.of( - ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg", - ".mp4", ".mov", ".mkv", ".webm", ".avi", - ".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a" - ); - - private final BackgroundTaskRepository backgroundTaskRepository; - private final StoredFileRepository storedFileRepository; - private final ObjectMapper objectMapper; - - @Transactional - public BackgroundTask createQueuedFileTask(User user, - BackgroundTaskType type, - Long fileId, - String requestedPath, - String correlationId) { - StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, user.getId()) - .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "file not found")); - String logicalPath = buildLogicalPath(file); - if (!logicalPath.equals(normalizeLogicalPath(requestedPath))) { - throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task path does not match file path"); - } - validateTaskTarget(type, file); - - Map publicState = fileState(file, logicalPath); - Map privateState = new LinkedHashMap<>(publicState); - privateState.put("taskType", type.name()); - return createQueuedTask(user, type, publicState, privateState, correlationId); - } - - @Transactional - public BackgroundTask createQueuedTask(User user, - BackgroundTaskType type, - Map publicState, - Map privateState, - String correlationId) { - BackgroundTask task = new BackgroundTask(); - task.setUserId(user.getId()); - task.setType(type); - task.setStatus(BackgroundTaskStatus.QUEUED); - task.setPublicStateJson(toJson(publicState)); - task.setPrivateStateJson(toJson(privateState)); - task.setCorrelationId(normalizeCorrelationId(correlationId)); - return backgroundTaskRepository.save(task); - } - - public Page listOwnedTasks(User user, Pageable pageable) { - return backgroundTaskRepository.findByUserIdOrderByCreatedAtDesc(user.getId(), pageable); - } - - public BackgroundTask getOwnedTask(User user, Long id) { - return backgroundTaskRepository.findByIdAndUserId(id, user.getId()) - .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found")); - } - - @Transactional - public BackgroundTask cancelOwnedTask(User user, Long id) { - BackgroundTask task = getOwnedTask(user, id); - if (task.isTerminal()) { - return task; - } - - if (task.getStatus() == BackgroundTaskStatus.QUEUED || task.getStatus() == BackgroundTaskStatus.RUNNING) { - task.setStatus(BackgroundTaskStatus.CANCELLED); - task.setFinishedAt(LocalDateTime.now()); - task.setErrorMessage(null); - return backgroundTaskRepository.save(task); - } - - return task; - } - - @Transactional - public BackgroundTask markRunning(User user, Long id) { - BackgroundTask task = getOwnedTask(user, id); - if (task.isTerminal()) { - return task; - } - task.setStatus(BackgroundTaskStatus.RUNNING); - return backgroundTaskRepository.save(task); - } - - @Transactional - public BackgroundTask markCompleted(User user, Long id) { - BackgroundTask task = getOwnedTask(user, id); - if (task.isTerminal()) { - return task; - } - task.setStatus(BackgroundTaskStatus.COMPLETED); - task.setFinishedAt(LocalDateTime.now()); - task.setErrorMessage(null); - return backgroundTaskRepository.save(task); - } - - @Transactional - public BackgroundTask markFailed(User user, Long id, String errorMessage) { - BackgroundTask task = getOwnedTask(user, id); - if (task.isTerminal()) { - return task; - } - task.setStatus(BackgroundTaskStatus.FAILED); - task.setFinishedAt(LocalDateTime.now()); - task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed"); - return backgroundTaskRepository.save(task); - } - - public List findQueuedTaskIds(int limit) { - if (limit <= 0) { - return List.of(); - } - - return backgroundTaskRepository.findByStatusOrderByCreatedAtAsc( - BackgroundTaskStatus.QUEUED, - PageRequest.of(0, limit) - ) - .stream() - .map(BackgroundTask::getId) - .toList(); - } - - @Transactional - public Optional claimQueuedTask(Long id) { - int claimed = backgroundTaskRepository.claimQueuedTask( - id, - BackgroundTaskStatus.QUEUED, - BackgroundTaskStatus.RUNNING, - LocalDateTime.now() - ); - if (claimed != 1) { - return Optional.empty(); - } - return backgroundTaskRepository.findById(id); - } - - @Transactional - public BackgroundTask markWorkerTaskCompleted(Long id, Map publicStatePatch) { - BackgroundTask task = backgroundTaskRepository.findById(id) - .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found")); - if (task.isTerminal() || task.getStatus() != BackgroundTaskStatus.RUNNING) { - return task; - } - - Map nextPublicState = parseJsonObject(task.getPublicStateJson()); - if (publicStatePatch != null) { - nextPublicState.putAll(publicStatePatch); - } - task.setPublicStateJson(toJson(nextPublicState)); - task.setStatus(BackgroundTaskStatus.COMPLETED); - task.setFinishedAt(LocalDateTime.now()); - task.setErrorMessage(null); - return backgroundTaskRepository.save(task); - } - - @Transactional - public BackgroundTask markWorkerTaskFailed(Long id, String errorMessage) { - BackgroundTask task = backgroundTaskRepository.findById(id) - .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found")); - if (task.isTerminal()) { - return task; - } - - task.setStatus(BackgroundTaskStatus.FAILED); - task.setFinishedAt(LocalDateTime.now()); - task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed"); - return backgroundTaskRepository.save(task); - } - - private String normalizeCorrelationId(String correlationId) { - if (StringUtils.hasText(correlationId)) { - return correlationId.trim(); - } - return UUID.randomUUID().toString().replace("-", ""); - } - - private void validateTaskTarget(BackgroundTaskType type, StoredFile file) { - if (type == BackgroundTaskType.ARCHIVE) { - return; - } - if (file.isDirectory()) { - throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task target type is not supported"); - } - if (type == BackgroundTaskType.EXTRACT && !isArchiveLike(file)) { - throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "extract task only supports archive files"); - } - if (type == BackgroundTaskType.MEDIA_META && !isMediaLike(file)) { - throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "media metadata task only supports media files"); - } - } - - private Map fileState(StoredFile file, String logicalPath) { - Map state = new LinkedHashMap<>(); - state.put("fileId", file.getId()); - state.put("path", logicalPath); - state.put("filename", file.getFilename()); - state.put("directory", file.isDirectory()); - state.put("contentType", file.getContentType()); - state.put("size", file.getSize()); - return state; - } - - private boolean isArchiveLike(StoredFile file) { - String contentType = normalizeContentType(file.getContentType()); - if (contentType.contains("zip") - || contentType.contains("x-tar") - || contentType.contains("gzip") - || contentType.contains("x-7z") - || contentType.contains("x-rar") - || contentType.contains("java-archive")) { - return true; - } - return hasExtension(file.getFilename(), ARCHIVE_EXTENSIONS); - } - - private boolean isMediaLike(StoredFile file) { - String contentType = normalizeContentType(file.getContentType()); - if (contentType.startsWith("image/") || contentType.startsWith("video/") || contentType.startsWith("audio/")) { - return true; - } - return hasExtension(file.getFilename(), MEDIA_EXTENSIONS); - } - - private boolean hasExtension(String filename, List extensions) { - if (!StringUtils.hasText(filename)) { - return false; - } - String normalized = filename.toLowerCase(Locale.ROOT); - return extensions.stream().anyMatch(normalized::endsWith); - } - - private String normalizeContentType(String contentType) { - if (!StringUtils.hasText(contentType)) { - return ""; - } - return contentType.trim().toLowerCase(Locale.ROOT); - } - - private String buildLogicalPath(StoredFile file) { - String parent = normalizeLogicalPath(file.getPath()); - if ("/".equals(parent)) { - return "/" + file.getFilename(); - } - return parent + "/" + file.getFilename(); - } - - private String normalizeLogicalPath(String path) { - if (!StringUtils.hasText(path)) { - return "/"; - } - String normalized = path.trim().replace('\\', '/'); - while (normalized.contains("//")) { - normalized = normalized.replace("//", "/"); - } - if (!normalized.startsWith("/")) { - normalized = "/" + normalized; - } - while (normalized.length() > 1 && normalized.endsWith("/")) { - normalized = normalized.substring(0, normalized.length() - 1); - } - return normalized; - } - - private String toJson(Map value) { - Map safeValue = value == null ? new LinkedHashMap<>() : new LinkedHashMap<>(value); - try { - return objectMapper.writeValueAsString(safeValue); - } catch (JsonProcessingException ex) { - throw new IllegalStateException("Failed to serialize background task state", ex); - } - } - - private Map parseJsonObject(String value) { - if (!StringUtils.hasText(value)) { - return new LinkedHashMap<>(); - } - - try { - return objectMapper.readValue(value, new TypeReference>() { - }); - } catch (JsonProcessingException ex) { - throw new IllegalStateException("Failed to parse background task state", ex); - } - } -} diff --git a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskWorker.java b/backend/src/main/java/com/yoyuzh/files/BackgroundTaskWorker.java deleted file mode 100644 index ef28527..0000000 --- a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskWorker.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.yoyuzh.files; - -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -public class BackgroundTaskWorker { - - private static final int DEFAULT_BATCH_SIZE = 5; - - private final BackgroundTaskService backgroundTaskService; - private final List handlers; - - public BackgroundTaskWorker(BackgroundTaskService backgroundTaskService, - List handlers) { - this.backgroundTaskService = backgroundTaskService; - this.handlers = List.copyOf(handlers); - } - - @Scheduled( - fixedDelayString = "${app.background-tasks.worker.fixed-delay-ms:30000}", - initialDelayString = "${app.background-tasks.worker.initial-delay-ms:30000}" - ) - public void runScheduledBatch() { - processQueuedTasks(DEFAULT_BATCH_SIZE); - } - - public int processQueuedTasks(int maxTasks) { - int processedCount = 0; - for (Long taskId : backgroundTaskService.findQueuedTaskIds(maxTasks)) { - var claimedTask = backgroundTaskService.claimQueuedTask(taskId); - if (claimedTask.isEmpty()) { - continue; - } - - execute(claimedTask.get()); - processedCount += 1; - } - return processedCount; - } - - private void execute(BackgroundTask task) { - try { - BackgroundTaskHandler handler = findHandler(task); - BackgroundTaskHandlerResult result = handler.handle(task); - backgroundTaskService.markWorkerTaskCompleted(task.getId(), result.publicStatePatch()); - } catch (Exception ex) { - String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage(); - backgroundTaskService.markWorkerTaskFailed(task.getId(), message); - } - } - - private BackgroundTaskHandler findHandler(BackgroundTask task) { - return handlers.stream() - .filter(handler -> handler.supports(task.getType())) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No background task handler for " + task.getType())); - } -} diff --git a/backend/src/main/java/com/yoyuzh/files/CopyFileRequest.java b/backend/src/main/java/com/yoyuzh/files/core/CopyFileRequest.java similarity index 84% rename from backend/src/main/java/com/yoyuzh/files/CopyFileRequest.java rename to backend/src/main/java/com/yoyuzh/files/core/CopyFileRequest.java index ef6d916..9c6ee75 100644 --- a/backend/src/main/java/com/yoyuzh/files/CopyFileRequest.java +++ b/backend/src/main/java/com/yoyuzh/files/core/CopyFileRequest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import jakarta.validation.constraints.NotBlank; diff --git a/backend/src/main/java/com/yoyuzh/files/DownloadUrlResponse.java b/backend/src/main/java/com/yoyuzh/files/core/DownloadUrlResponse.java similarity index 62% rename from backend/src/main/java/com/yoyuzh/files/DownloadUrlResponse.java rename to backend/src/main/java/com/yoyuzh/files/core/DownloadUrlResponse.java index 8a86a7a..cde8be0 100644 --- a/backend/src/main/java/com/yoyuzh/files/DownloadUrlResponse.java +++ b/backend/src/main/java/com/yoyuzh/files/core/DownloadUrlResponse.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; public record DownloadUrlResponse(String url) { } diff --git a/backend/src/main/java/com/yoyuzh/files/FileBlob.java b/backend/src/main/java/com/yoyuzh/files/core/FileBlob.java similarity index 98% rename from backend/src/main/java/com/yoyuzh/files/FileBlob.java rename to backend/src/main/java/com/yoyuzh/files/core/FileBlob.java index a9d52af..a8d0fec 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileBlob.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileBlob.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import jakarta.persistence.Column; import jakarta.persistence.Entity; diff --git a/backend/src/main/java/com/yoyuzh/files/FileBlobBackfillService.java b/backend/src/main/java/com/yoyuzh/files/core/FileBlobBackfillService.java similarity index 98% rename from backend/src/main/java/com/yoyuzh/files/FileBlobBackfillService.java rename to backend/src/main/java/com/yoyuzh/files/core/FileBlobBackfillService.java index 4805be2..193bc48 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileBlobBackfillService.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileBlobBackfillService.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.files.storage.FileContentStorage; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/yoyuzh/files/FileBlobRepository.java b/backend/src/main/java/com/yoyuzh/files/core/FileBlobRepository.java similarity index 93% rename from backend/src/main/java/com/yoyuzh/files/FileBlobRepository.java rename to backend/src/main/java/com/yoyuzh/files/core/FileBlobRepository.java index df3aeab..9ad3bd8 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileBlobRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileBlobRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; diff --git a/backend/src/main/java/com/yoyuzh/files/FileController.java b/backend/src/main/java/com/yoyuzh/files/core/FileController.java similarity index 96% rename from backend/src/main/java/com/yoyuzh/files/FileController.java rename to backend/src/main/java/com/yoyuzh/files/core/FileController.java index 2a90699..ef17f5d 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileController.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileController.java @@ -1,8 +1,14 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.common.ApiResponse; import com.yoyuzh.common.PageResponse; +import com.yoyuzh.files.share.CreateFileShareLinkResponse; +import com.yoyuzh.files.share.FileShareDetailsResponse; +import com.yoyuzh.files.share.ImportSharedFileRequest; +import com.yoyuzh.files.upload.CompleteUploadRequest; +import com.yoyuzh.files.upload.InitiateUploadRequest; +import com.yoyuzh.files.upload.InitiateUploadResponse; import io.swagger.v3.oas.annotations.Operation; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/yoyuzh/files/FileEntity.java b/backend/src/main/java/com/yoyuzh/files/core/FileEntity.java similarity index 99% rename from backend/src/main/java/com/yoyuzh/files/FileEntity.java rename to backend/src/main/java/com/yoyuzh/files/core/FileEntity.java index 5f19643..8940757 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileEntity.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileEntity.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.auth.User; import jakarta.persistence.Column; diff --git a/backend/src/main/java/com/yoyuzh/files/FileEntityBackfillService.java b/backend/src/main/java/com/yoyuzh/files/core/FileEntityBackfillService.java similarity index 96% rename from backend/src/main/java/com/yoyuzh/files/FileEntityBackfillService.java rename to backend/src/main/java/com/yoyuzh/files/core/FileEntityBackfillService.java index c489d52..60cd696 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileEntityBackfillService.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileEntityBackfillService.java @@ -1,5 +1,6 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; +import com.yoyuzh.files.policy.StoragePolicyService; import lombok.RequiredArgsConstructor; import org.springframework.boot.CommandLineRunner; import org.springframework.core.annotation.Order; diff --git a/backend/src/main/java/com/yoyuzh/files/FileEntityRepository.java b/backend/src/main/java/com/yoyuzh/files/core/FileEntityRepository.java similarity index 89% rename from backend/src/main/java/com/yoyuzh/files/FileEntityRepository.java rename to backend/src/main/java/com/yoyuzh/files/core/FileEntityRepository.java index ee073d3..cf0295c 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileEntityRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileEntityRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/backend/src/main/java/com/yoyuzh/files/FileEntityType.java b/backend/src/main/java/com/yoyuzh/files/core/FileEntityType.java similarity index 76% rename from backend/src/main/java/com/yoyuzh/files/FileEntityType.java rename to backend/src/main/java/com/yoyuzh/files/core/FileEntityType.java index b961550..b65546c 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileEntityType.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileEntityType.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; public enum FileEntityType { VERSION, diff --git a/backend/src/main/java/com/yoyuzh/files/FileMetadataResponse.java b/backend/src/main/java/com/yoyuzh/files/core/FileMetadataResponse.java similarity index 88% rename from backend/src/main/java/com/yoyuzh/files/FileMetadataResponse.java rename to backend/src/main/java/com/yoyuzh/files/core/FileMetadataResponse.java index 85ba4fb..c471a4e 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileMetadataResponse.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileMetadataResponse.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import java.time.LocalDateTime; diff --git a/backend/src/main/java/com/yoyuzh/files/FileService.java b/backend/src/main/java/com/yoyuzh/files/core/FileService.java similarity index 74% rename from backend/src/main/java/com/yoyuzh/files/FileService.java rename to backend/src/main/java/com/yoyuzh/files/core/FileService.java index 6621bfe..3ac57f1 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileService.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileService.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.admin.AdminMetricsService; import com.yoyuzh.auth.User; @@ -6,8 +6,18 @@ import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.PageResponse; import com.yoyuzh.config.FileStorageProperties; +import com.yoyuzh.files.events.FileEventService; +import com.yoyuzh.files.events.FileEventType; +import com.yoyuzh.files.policy.StoragePolicyService; +import com.yoyuzh.files.share.CreateFileShareLinkResponse; +import com.yoyuzh.files.share.FileShareDetailsResponse; +import com.yoyuzh.files.share.FileShareLink; +import com.yoyuzh.files.share.FileShareLinkRepository; import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.PreparedUpload; +import com.yoyuzh.files.upload.CompleteUploadRequest; +import com.yoyuzh.files.upload.InitiateUploadRequest; +import com.yoyuzh.files.upload.InitiateUploadResponse; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -20,6 +30,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; import org.springframework.web.multipart.MultipartFile; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URI; @@ -41,6 +52,7 @@ import java.util.Set; import java.util.UUID; import java.util.Locale; import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @Service @@ -566,36 +578,49 @@ public class FileService { }); } - private ResponseEntity downloadDirectory(User user, StoredFile directory) { - String logicalPath = buildLogicalPath(directory); - String archiveName = directory.getFilename() + ".zip"; - List descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(user.getId(), logicalPath) - .stream() - .sorted(Comparator.comparing(StoredFile::getPath).thenComparing(StoredFile::getFilename)) - .toList(); + @Transactional + public void importExternalFilesAtomically(User recipient, + List directories, + List files) { + importExternalFilesAtomically(recipient, directories, files, null); + } - byte[] archiveBytes; - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) { - Set createdEntries = new LinkedHashSet<>(); - writeDirectoryEntry(zipOutputStream, createdEntries, directory.getFilename() + "/"); + @Transactional + public void importExternalFilesAtomically(User recipient, + List directories, + List files, + ExternalImportProgressListener progressListener) { + List normalizedDirectories = normalizeExternalImportDirectories(directories); + List normalizedFiles = normalizeExternalImportFiles(files); + validateExternalImportBatch(recipient, normalizedDirectories, normalizedFiles); - for (StoredFile descendant : descendants) { - String entryName = buildZipEntryName(directory.getFilename(), logicalPath, descendant); - if (descendant.isDirectory()) { - writeDirectoryEntry(zipOutputStream, createdEntries, entryName + "/"); - continue; - } - - ensureParentDirectoryEntries(zipOutputStream, createdEntries, entryName); - writeFileEntry(zipOutputStream, createdEntries, entryName, - fileContentStorage.readBlob(getRequiredBlob(descendant).getObjectKey())); + List writtenBlobObjectKeys = new ArrayList<>(); + int totalDirectoryCount = normalizedDirectories.size(); + int totalFileCount = normalizedFiles.size(); + int processedDirectoryCount = 0; + int processedFileCount = 0; + try { + for (String directory : normalizedDirectories) { + mkdir(recipient, directory); + processedDirectoryCount += 1; + reportExternalImportProgress(progressListener, processedFileCount, totalFileCount, + processedDirectoryCount, totalDirectoryCount); } - zipOutputStream.finish(); - archiveBytes = outputStream.toByteArray(); - } catch (IOException ex) { - throw new BusinessException(ErrorCode.UNKNOWN, "目录压缩失败"); + for (ExternalFileImport file : normalizedFiles) { + storeExternalImportFile(recipient, file, writtenBlobObjectKeys); + processedFileCount += 1; + reportExternalImportProgress(progressListener, processedFileCount, totalFileCount, + processedDirectoryCount, totalDirectoryCount); + } + } catch (RuntimeException ex) { + cleanupWrittenBlobs(writtenBlobObjectKeys, ex); + throw ex; } + } + + private ResponseEntity downloadDirectory(User user, StoredFile directory) { + String archiveName = directory.getFilename() + ".zip"; + byte[] archiveBytes = buildArchiveBytes(directory); return ResponseEntity.ok() .header(HttpHeaders.CONTENT_DISPOSITION, @@ -604,6 +629,63 @@ public class FileService { .body(archiveBytes); } + public byte[] buildArchiveBytes(StoredFile source) { + return buildArchiveBytes(source, null); + } + + public byte[] buildArchiveBytes(StoredFile source, ArchiveBuildProgressListener progressListener) { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) { + Set createdEntries = new LinkedHashSet<>(); + ArchiveBuildProgressState progressState = createArchiveBuildProgressState(source, progressListener); + reportArchiveProgress(progressState); + if (source.isDirectory()) { + writeDirectoryArchiveEntries(zipOutputStream, createdEntries, source, progressState); + } else { + writeFileArchiveEntry(zipOutputStream, createdEntries, source.getFilename(), source, progressState); + } + zipOutputStream.finish(); + return outputStream.toByteArray(); + } catch (IOException ex) { + throw new BusinessException(ErrorCode.UNKNOWN, "目录压缩失败"); + } + } + + public ZipCompatibleArchive readZipCompatibleArchive(StoredFile source) { + byte[] archiveBytes = fileContentStorage.readBlob(getRequiredBlob(source).getObjectKey()); + try (ZipInputStream zipInputStream = new ZipInputStream( + new ByteArrayInputStream(archiveBytes), + StandardCharsets.UTF_8)) { + List entries = new ArrayList<>(); + Map seenEntries = new HashMap<>(); + ZipEntry entry = zipInputStream.getNextEntry(); + while (entry != null) { + String relativePath = normalizeZipCompatibleEntryPath(entry.getName()); + if (StringUtils.hasText(relativePath)) { + boolean directory = entry.isDirectory() || entry.getName().endsWith("/"); + Boolean existingType = seenEntries.putIfAbsent(relativePath, directory); + if (existingType != null) { + throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法"); + } + entries.add(new ZipCompatibleArchiveEntry( + relativePath, + directory, + directory ? new byte[0] : zipInputStream.readAllBytes() + )); + } + entry = zipInputStream.getNextEntry(); + } + if (entries.isEmpty() && !hasZipCompatibleSignature(archiveBytes)) { + throw new BusinessException(ErrorCode.UNKNOWN, "压缩包读取失败"); + } + return new ZipCompatibleArchive(entries, detectCommonRootDirectoryName(entries)); + } catch (BusinessException ex) { + throw ex; + } catch (IOException ex) { + throw new BusinessException(ErrorCode.UNKNOWN, "压缩包读取失败"); + } + } + private boolean shouldUsePublicPackageDownload(StoredFile storedFile) { return fileContentStorage.supportsDirectDownload() && StringUtils.hasText(packageDownloadBaseUrl) @@ -854,6 +936,62 @@ public class FileService { ensureWithinStorageQuota(user, size); } + private List normalizeExternalImportDirectories(List directories) { + if (directories == null || directories.isEmpty()) { + return List.of(); + } + return directories.stream() + .map(this::normalizeDirectoryPath) + .distinct() + .sorted(Comparator.comparingInt(String::length).thenComparing(String::compareTo)) + .toList(); + } + + private List normalizeExternalImportFiles(List files) { + if (files == null || files.isEmpty()) { + return List.of(); + } + return files.stream() + .map(file -> new ExternalFileImport( + normalizeDirectoryPath(file.path()), + normalizeLeafName(file.filename()), + StringUtils.hasText(file.contentType()) ? file.contentType().trim() : "application/octet-stream", + file.content() == null ? new byte[0] : file.content() + )) + .toList(); + } + + private void validateExternalImportBatch(User recipient, + List directories, + List files) { + ensureWithinStorageQuota(recipient, files.stream().mapToLong(ExternalFileImport::size).sum()); + + Set plannedTargets = new LinkedHashSet<>(); + for (String directory : directories) { + if ("/".equals(directory)) { + continue; + } + if (!plannedTargets.add(directory)) { + continue; + } + String parentPath = extractParentPath(directory); + String directoryName = extractLeafName(directory); + if (storedFileRepository.existsByUserIdAndPathAndFilename(recipient.getId(), parentPath, directoryName)) { + throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在"); + } + } + + for (ExternalFileImport file : files) { + String logicalPath = buildTargetLogicalPath(file.path(), file.filename()); + if (plannedTargets.contains(logicalPath) || !plannedTargets.add(logicalPath)) { + throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在"); + } + if (storedFileRepository.existsByUserIdAndPathAndFilename(recipient.getId(), file.path(), file.filename())) { + throw new BusinessException(ErrorCode.UNKNOWN, "同目录下文件已存在"); + } + } + } + private void ensureWithinStorageQuota(User user, long additionalBytes) { if (additionalBytes <= 0) { return; @@ -900,6 +1038,25 @@ public class FileService { } } + private void storeExternalImportFile(User recipient, + ExternalFileImport file, + List writtenBlobObjectKeys) { + validateUpload(recipient, file.path(), file.filename(), file.size()); + ensureDirectoryHierarchy(recipient, file.path()); + String objectKey = createBlobObjectKey(); + writtenBlobObjectKeys.add(objectKey); + fileContentStorage.storeBlob(objectKey, file.contentType(), file.content()); + FileBlob blob = createAndSaveBlob(objectKey, file.contentType(), file.size()); + saveFileMetadata( + recipient, + file.path(), + file.filename(), + file.contentType(), + file.size(), + blob + ); + } + private void moveToRecycleBin(List filesToRecycle, Long recycleRootId) { if (filesToRecycle.isEmpty()) { return; @@ -1071,6 +1228,37 @@ public class FileService { return savedFile; } + private void writeDirectoryArchiveEntries(ZipOutputStream zipOutputStream, + Set createdEntries, + StoredFile directory, + ArchiveBuildProgressState progressState) throws IOException { + String logicalPath = buildLogicalPath(directory); + List descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(directory.getUser().getId(), logicalPath) + .stream() + .sorted(Comparator.comparing(StoredFile::getPath).thenComparing(StoredFile::getFilename)) + .toList(); + writeDirectoryEntry(zipOutputStream, createdEntries, directory.getFilename() + "/", progressState); + + for (StoredFile descendant : descendants) { + String entryName = buildZipEntryName(directory.getFilename(), logicalPath, descendant); + if (descendant.isDirectory()) { + writeDirectoryEntry(zipOutputStream, createdEntries, entryName + "/", progressState); + continue; + } + writeFileArchiveEntry(zipOutputStream, createdEntries, entryName, descendant, progressState); + } + } + + private void writeFileArchiveEntry(ZipOutputStream zipOutputStream, + Set createdEntries, + String entryName, + StoredFile file, + ArchiveBuildProgressState progressState) throws IOException { + ensureParentDirectoryEntries(zipOutputStream, createdEntries, entryName, progressState); + writeFileEntry(zipOutputStream, createdEntries, entryName, progressState, + fileContentStorage.readBlob(getRequiredBlob(file).getObjectKey())); + } + private String buildZipEntryName(String rootDirectoryName, String rootLogicalPath, StoredFile storedFile) { StringBuilder entryName = new StringBuilder(rootDirectoryName).append('/'); if (!storedFile.getPath().equals(rootLogicalPath)) { @@ -1080,24 +1268,153 @@ public class FileService { return entryName.toString(); } - private void ensureParentDirectoryEntries(ZipOutputStream zipOutputStream, Set createdEntries, String entryName) throws IOException { + private String normalizeZipCompatibleEntryPath(String entryName) { + String normalized = entryName == null ? "" : entryName.trim().replace("\\", "/"); + if (!StringUtils.hasText(normalized)) { + return ""; + } + if (normalized.startsWith("/")) { + throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法"); + } + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + if (!StringUtils.hasText(normalized)) { + return ""; + } + + StringBuilder sanitized = new StringBuilder(); + for (String segment : normalized.split("/")) { + if (!StringUtils.hasText(segment) || ".".equals(segment) || "..".equals(segment)) { + throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法"); + } + if (sanitized.length() > 0) { + sanitized.append('/'); + } + sanitized.append(normalizeLeafName(segment)); + } + return sanitized.toString(); + } + + private String detectCommonRootDirectoryName(List entries) { + String candidate = null; + boolean hasNestedEntry = false; + boolean hasDirectoryCandidate = false; + for (ZipCompatibleArchiveEntry entry : entries) { + String relativePath = entry.relativePath(); + int slashIndex = relativePath.indexOf('/'); + String topSegment = slashIndex >= 0 ? relativePath.substring(0, slashIndex) : relativePath; + if (candidate == null) { + candidate = topSegment; + } else if (!candidate.equals(topSegment)) { + return null; + } + if (slashIndex >= 0) { + hasNestedEntry = true; + } + if (entry.directory() && candidate.equals(relativePath)) { + hasDirectoryCandidate = true; + } + if (!entry.directory() && candidate.equals(relativePath)) { + return null; + } + } + if (!hasNestedEntry && !hasDirectoryCandidate) { + return null; + } + return candidate; + } + + private boolean hasZipCompatibleSignature(byte[] archiveBytes) { + if (archiveBytes == null || archiveBytes.length < 4) { + return false; + } + return archiveBytes[0] == 'P' + && archiveBytes[1] == 'K' + && (archiveBytes[2] == 3 || archiveBytes[2] == 5 || archiveBytes[2] == 7) + && (archiveBytes[3] == 4 || archiveBytes[3] == 6 || archiveBytes[3] == 8); + } + + public ArchiveSourceSummary summarizeArchiveSource(StoredFile source) { + if (!source.isDirectory()) { + return new ArchiveSourceSummary(1, 0); + } + String logicalPath = buildLogicalPath(source); + List descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(source.getUser().getId(), logicalPath); + int directoryCount = 1 + (int) descendants.stream().filter(StoredFile::isDirectory).count(); + int fileCount = (int) descendants.stream().filter(file -> !file.isDirectory()).count(); + return new ArchiveSourceSummary(fileCount, directoryCount); + } + + private ArchiveBuildProgressState createArchiveBuildProgressState(StoredFile source, + ArchiveBuildProgressListener progressListener) { + if (progressListener == null) { + return null; + } + ArchiveSourceSummary summary = summarizeArchiveSource(source); + return new ArchiveBuildProgressState(progressListener, summary.fileCount(), summary.directoryCount()); + } + + private void reportArchiveProgress(ArchiveBuildProgressState progressState) { + if (progressState == null) { + return; + } + progressState.listener.onProgress(new ArchiveBuildProgress( + progressState.processedFileCount, + progressState.totalFileCount, + progressState.processedDirectoryCount, + progressState.totalDirectoryCount + )); + } + + private void reportExternalImportProgress(ExternalImportProgressListener progressListener, + int processedFileCount, + int totalFileCount, + int processedDirectoryCount, + int totalDirectoryCount) { + if (progressListener == null) { + return; + } + progressListener.onProgress(new ExternalImportProgress( + processedFileCount, + totalFileCount, + processedDirectoryCount, + totalDirectoryCount + )); + } + + private void ensureParentDirectoryEntries(ZipOutputStream zipOutputStream, + Set createdEntries, + String entryName, + ArchiveBuildProgressState progressState) throws IOException { int slashIndex = entryName.indexOf('/'); while (slashIndex >= 0) { - writeDirectoryEntry(zipOutputStream, createdEntries, entryName.substring(0, slashIndex + 1)); + writeDirectoryEntry(zipOutputStream, createdEntries, entryName.substring(0, slashIndex + 1), progressState); slashIndex = entryName.indexOf('/', slashIndex + 1); } } - private void writeDirectoryEntry(ZipOutputStream zipOutputStream, Set createdEntries, String entryName) throws IOException { + private void writeDirectoryEntry(ZipOutputStream zipOutputStream, + Set createdEntries, + String entryName, + ArchiveBuildProgressState progressState) throws IOException { if (!createdEntries.add(entryName)) { return; } zipOutputStream.putNextEntry(new ZipEntry(entryName)); zipOutputStream.closeEntry(); + if (progressState != null) { + progressState.processedDirectoryCount += 1; + reportArchiveProgress(progressState); + } } - private void writeFileEntry(ZipOutputStream zipOutputStream, Set createdEntries, String entryName, byte[] content) + private void writeFileEntry(ZipOutputStream zipOutputStream, + Set createdEntries, + String entryName, + ArchiveBuildProgressState progressState, + byte[] content) throws IOException { if (!createdEntries.add(entryName)) { return; @@ -1106,6 +1423,10 @@ public class FileService { zipOutputStream.putNextEntry(new ZipEntry(entryName)); zipOutputStream.write(content); zipOutputStream.closeEntry(); + if (progressState != null) { + progressState.processedFileCount += 1; + reportArchiveProgress(progressState); + } } private void recordFileEvent(User user, @@ -1170,6 +1491,16 @@ public class FileService { } } + private void cleanupWrittenBlobs(List writtenBlobObjectKeys, RuntimeException ex) { + for (String objectKey : writtenBlobObjectKeys) { + try { + fileContentStorage.deleteBlob(objectKey); + } catch (RuntimeException cleanupEx) { + ex.addSuppressed(cleanupEx); + } + } + } + private FileBlob createAndSaveBlob(String objectKey, String contentType, long size) { FileBlob blob = new FileBlob(); blob.setObjectKey(objectKey); @@ -1248,4 +1579,57 @@ public class FileService { private interface BlobWriteOperation { T run(); } + + public static record ZipCompatibleArchive(List entries, String commonRootDirectoryName) { + } + + public static record ZipCompatibleArchiveEntry(String relativePath, boolean directory, byte[] content) { + } + + public static record ExternalFileImport(String path, String filename, String contentType, byte[] content) { + public long size() { + return content == null ? 0L : content.length; + } + } + + public record ArchiveSourceSummary(int fileCount, int directoryCount) { + } + + public record ArchiveBuildProgress(int processedFileCount, + int totalFileCount, + int processedDirectoryCount, + int totalDirectoryCount) { + } + + @FunctionalInterface + public interface ArchiveBuildProgressListener { + void onProgress(ArchiveBuildProgress progress); + } + + public record ExternalImportProgress(int processedFileCount, + int totalFileCount, + int processedDirectoryCount, + int totalDirectoryCount) { + } + + @FunctionalInterface + public interface ExternalImportProgressListener { + void onProgress(ExternalImportProgress progress); + } + + private static final class ArchiveBuildProgressState { + private final ArchiveBuildProgressListener listener; + private final int totalFileCount; + private final int totalDirectoryCount; + private int processedFileCount; + private int processedDirectoryCount; + + private ArchiveBuildProgressState(ArchiveBuildProgressListener listener, + int totalFileCount, + int totalDirectoryCount) { + this.listener = listener; + this.totalFileCount = totalFileCount; + this.totalDirectoryCount = totalDirectoryCount; + } + } } diff --git a/backend/src/main/java/com/yoyuzh/files/MkdirRequest.java b/backend/src/main/java/com/yoyuzh/files/core/MkdirRequest.java similarity index 77% rename from backend/src/main/java/com/yoyuzh/files/MkdirRequest.java rename to backend/src/main/java/com/yoyuzh/files/core/MkdirRequest.java index cd75952..2427121 100644 --- a/backend/src/main/java/com/yoyuzh/files/MkdirRequest.java +++ b/backend/src/main/java/com/yoyuzh/files/core/MkdirRequest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import jakarta.validation.constraints.NotBlank; diff --git a/backend/src/main/java/com/yoyuzh/files/MoveFileRequest.java b/backend/src/main/java/com/yoyuzh/files/core/MoveFileRequest.java similarity index 84% rename from backend/src/main/java/com/yoyuzh/files/MoveFileRequest.java rename to backend/src/main/java/com/yoyuzh/files/core/MoveFileRequest.java index b2e9e2d..48f87ea 100644 --- a/backend/src/main/java/com/yoyuzh/files/MoveFileRequest.java +++ b/backend/src/main/java/com/yoyuzh/files/core/MoveFileRequest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import jakarta.validation.constraints.NotBlank; diff --git a/backend/src/main/java/com/yoyuzh/files/RecycleBinItemResponse.java b/backend/src/main/java/com/yoyuzh/files/core/RecycleBinItemResponse.java similarity index 90% rename from backend/src/main/java/com/yoyuzh/files/RecycleBinItemResponse.java rename to backend/src/main/java/com/yoyuzh/files/core/RecycleBinItemResponse.java index 83e500b..0993028 100644 --- a/backend/src/main/java/com/yoyuzh/files/RecycleBinItemResponse.java +++ b/backend/src/main/java/com/yoyuzh/files/core/RecycleBinItemResponse.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import java.time.LocalDateTime; diff --git a/backend/src/main/java/com/yoyuzh/files/RenameFileRequest.java b/backend/src/main/java/com/yoyuzh/files/core/RenameFileRequest.java similarity index 84% rename from backend/src/main/java/com/yoyuzh/files/RenameFileRequest.java rename to backend/src/main/java/com/yoyuzh/files/core/RenameFileRequest.java index 0b61fa1..b262e08 100644 --- a/backend/src/main/java/com/yoyuzh/files/RenameFileRequest.java +++ b/backend/src/main/java/com/yoyuzh/files/core/RenameFileRequest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import jakarta.validation.constraints.NotBlank; diff --git a/backend/src/main/java/com/yoyuzh/files/StoredFile.java b/backend/src/main/java/com/yoyuzh/files/core/StoredFile.java similarity index 99% rename from backend/src/main/java/com/yoyuzh/files/StoredFile.java rename to backend/src/main/java/com/yoyuzh/files/core/StoredFile.java index e68f1e3..4f84448 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoredFile.java +++ b/backend/src/main/java/com/yoyuzh/files/core/StoredFile.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.auth.User; import jakarta.persistence.Column; diff --git a/backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java b/backend/src/main/java/com/yoyuzh/files/core/StoredFileEntity.java similarity index 98% rename from backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java rename to backend/src/main/java/com/yoyuzh/files/core/StoredFileEntity.java index 8c1ce28..2a865ab 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java +++ b/backend/src/main/java/com/yoyuzh/files/core/StoredFileEntity.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import jakarta.persistence.Column; import jakarta.persistence.Entity; diff --git a/backend/src/main/java/com/yoyuzh/files/StoredFileEntityRepository.java b/backend/src/main/java/com/yoyuzh/files/core/StoredFileEntityRepository.java similarity index 83% rename from backend/src/main/java/com/yoyuzh/files/StoredFileEntityRepository.java rename to backend/src/main/java/com/yoyuzh/files/core/StoredFileEntityRepository.java index e63b4c9..85c1149 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoredFileEntityRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/core/StoredFileEntityRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/backend/src/main/java/com/yoyuzh/files/StoredFileRepository.java b/backend/src/main/java/com/yoyuzh/files/core/StoredFileRepository.java similarity index 99% rename from backend/src/main/java/com/yoyuzh/files/StoredFileRepository.java rename to backend/src/main/java/com/yoyuzh/files/core/StoredFileRepository.java index e1f9752..4e4b232 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoredFileRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/core/StoredFileRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; diff --git a/backend/src/main/java/com/yoyuzh/files/FileEvent.java b/backend/src/main/java/com/yoyuzh/files/events/FileEvent.java similarity index 98% rename from backend/src/main/java/com/yoyuzh/files/FileEvent.java rename to backend/src/main/java/com/yoyuzh/files/events/FileEvent.java index 3cebd48..56fd536 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileEvent.java +++ b/backend/src/main/java/com/yoyuzh/files/events/FileEvent.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.events; import jakarta.persistence.Column; import jakarta.persistence.Entity; diff --git a/backend/src/main/java/com/yoyuzh/files/FileEventRepository.java b/backend/src/main/java/com/yoyuzh/files/events/FileEventRepository.java similarity index 81% rename from backend/src/main/java/com/yoyuzh/files/FileEventRepository.java rename to backend/src/main/java/com/yoyuzh/files/events/FileEventRepository.java index ccb6637..346c16e 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileEventRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/events/FileEventRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.events; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/backend/src/main/java/com/yoyuzh/files/FileEventService.java b/backend/src/main/java/com/yoyuzh/files/events/FileEventService.java similarity index 99% rename from backend/src/main/java/com/yoyuzh/files/FileEventService.java rename to backend/src/main/java/com/yoyuzh/files/events/FileEventService.java index b1caffd..c413c25 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileEventService.java +++ b/backend/src/main/java/com/yoyuzh/files/events/FileEventService.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.events; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/backend/src/main/java/com/yoyuzh/files/FileEventType.java b/backend/src/main/java/com/yoyuzh/files/events/FileEventType.java similarity index 76% rename from backend/src/main/java/com/yoyuzh/files/FileEventType.java rename to backend/src/main/java/com/yoyuzh/files/events/FileEventType.java index 1c2ad03..bce6d64 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileEventType.java +++ b/backend/src/main/java/com/yoyuzh/files/events/FileEventType.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.events; public enum FileEventType { CREATED, diff --git a/backend/src/main/java/com/yoyuzh/files/StoragePolicy.java b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicy.java similarity index 99% rename from backend/src/main/java/com/yoyuzh/files/StoragePolicy.java rename to backend/src/main/java/com/yoyuzh/files/policy/StoragePolicy.java index eca58da..0746083 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoragePolicy.java +++ b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicy.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.policy; import jakarta.persistence.Column; import jakarta.persistence.Entity; diff --git a/backend/src/main/java/com/yoyuzh/files/StoragePolicyCapabilities.java b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyCapabilities.java similarity index 91% rename from backend/src/main/java/com/yoyuzh/files/StoragePolicyCapabilities.java rename to backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyCapabilities.java index 82e3251..cda9aef 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoragePolicyCapabilities.java +++ b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyCapabilities.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.policy; public record StoragePolicyCapabilities( boolean directUpload, diff --git a/backend/src/main/java/com/yoyuzh/files/StoragePolicyCredentialMode.java b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyCredentialMode.java similarity index 72% rename from backend/src/main/java/com/yoyuzh/files/StoragePolicyCredentialMode.java rename to backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyCredentialMode.java index 38d32a9..d5816f9 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoragePolicyCredentialMode.java +++ b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyCredentialMode.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.policy; public enum StoragePolicyCredentialMode { NONE, diff --git a/backend/src/main/java/com/yoyuzh/files/StoragePolicyRepository.java b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyRepository.java similarity index 88% rename from backend/src/main/java/com/yoyuzh/files/StoragePolicyRepository.java rename to backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyRepository.java index 7d395cd..ce6a875 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoragePolicyRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.policy; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/backend/src/main/java/com/yoyuzh/files/StoragePolicyService.java b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyService.java similarity index 98% rename from backend/src/main/java/com/yoyuzh/files/StoragePolicyService.java rename to backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyService.java index ef846f9..5df6707 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoragePolicyService.java +++ b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyService.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.policy; import com.fasterxml.jackson.databind.ObjectMapper; import com.yoyuzh.config.FileStorageProperties; @@ -57,7 +57,7 @@ public class StoragePolicyService implements CommandLineRunner { policy.setMaxSizeBytes(properties.getMaxFileSize()); policy.setCapabilitiesJson(writeCapabilities(new StoragePolicyCapabilities( true, - false, + true, true, true, false, diff --git a/backend/src/main/java/com/yoyuzh/files/StoragePolicyType.java b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyType.java similarity index 65% rename from backend/src/main/java/com/yoyuzh/files/StoragePolicyType.java rename to backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyType.java index 869180b..64625b4 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoragePolicyType.java +++ b/backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyType.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.policy; public enum StoragePolicyType { LOCAL, diff --git a/backend/src/main/java/com/yoyuzh/files/FileMetadata.java b/backend/src/main/java/com/yoyuzh/files/search/FileMetadata.java similarity index 97% rename from backend/src/main/java/com/yoyuzh/files/FileMetadata.java rename to backend/src/main/java/com/yoyuzh/files/search/FileMetadata.java index bce7d36..d45f7c0 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileMetadata.java +++ b/backend/src/main/java/com/yoyuzh/files/search/FileMetadata.java @@ -1,5 +1,6 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.search; +import com.yoyuzh.files.core.StoredFile; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.FetchType; diff --git a/backend/src/main/java/com/yoyuzh/files/FileMetadataRepository.java b/backend/src/main/java/com/yoyuzh/files/search/FileMetadataRepository.java similarity index 88% rename from backend/src/main/java/com/yoyuzh/files/FileMetadataRepository.java rename to backend/src/main/java/com/yoyuzh/files/search/FileMetadataRepository.java index a975074..ba3d339 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileMetadataRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/search/FileMetadataRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.search; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/backend/src/main/java/com/yoyuzh/files/FileSearchQuery.java b/backend/src/main/java/com/yoyuzh/files/search/FileSearchQuery.java similarity index 91% rename from backend/src/main/java/com/yoyuzh/files/FileSearchQuery.java rename to backend/src/main/java/com/yoyuzh/files/search/FileSearchQuery.java index cfa9848..df105ce 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileSearchQuery.java +++ b/backend/src/main/java/com/yoyuzh/files/search/FileSearchQuery.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.search; import java.time.LocalDateTime; diff --git a/backend/src/main/java/com/yoyuzh/files/FileSearchService.java b/backend/src/main/java/com/yoyuzh/files/search/FileSearchService.java similarity index 95% rename from backend/src/main/java/com/yoyuzh/files/FileSearchService.java rename to backend/src/main/java/com/yoyuzh/files/search/FileSearchService.java index 6c4fc0f..37d0b58 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileSearchService.java +++ b/backend/src/main/java/com/yoyuzh/files/search/FileSearchService.java @@ -1,9 +1,12 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.search; import com.yoyuzh.api.v2.ApiV2ErrorCode; import com.yoyuzh.api.v2.ApiV2Exception; import com.yoyuzh.auth.User; import com.yoyuzh.common.PageResponse; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; import lombok.RequiredArgsConstructor; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; diff --git a/backend/src/main/java/com/yoyuzh/files/CreateFileShareLinkResponse.java b/backend/src/main/java/com/yoyuzh/files/share/CreateFileShareLinkResponse.java similarity index 86% rename from backend/src/main/java/com/yoyuzh/files/CreateFileShareLinkResponse.java rename to backend/src/main/java/com/yoyuzh/files/share/CreateFileShareLinkResponse.java index 1fa12a7..23601bd 100644 --- a/backend/src/main/java/com/yoyuzh/files/CreateFileShareLinkResponse.java +++ b/backend/src/main/java/com/yoyuzh/files/share/CreateFileShareLinkResponse.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.share; import java.time.LocalDateTime; diff --git a/backend/src/main/java/com/yoyuzh/files/FileShareDetailsResponse.java b/backend/src/main/java/com/yoyuzh/files/share/FileShareDetailsResponse.java similarity index 89% rename from backend/src/main/java/com/yoyuzh/files/FileShareDetailsResponse.java rename to backend/src/main/java/com/yoyuzh/files/share/FileShareDetailsResponse.java index a5e7678..d421282 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileShareDetailsResponse.java +++ b/backend/src/main/java/com/yoyuzh/files/share/FileShareDetailsResponse.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.share; import java.time.LocalDateTime; diff --git a/backend/src/main/java/com/yoyuzh/files/FileShareLink.java b/backend/src/main/java/com/yoyuzh/files/share/FileShareLink.java similarity index 98% rename from backend/src/main/java/com/yoyuzh/files/FileShareLink.java rename to backend/src/main/java/com/yoyuzh/files/share/FileShareLink.java index c96a21b..80128c7 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileShareLink.java +++ b/backend/src/main/java/com/yoyuzh/files/share/FileShareLink.java @@ -1,6 +1,7 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.share; import com.yoyuzh.auth.User; +import com.yoyuzh.files.core.StoredFile; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.FetchType; diff --git a/backend/src/main/java/com/yoyuzh/files/FileShareLinkRepository.java b/backend/src/main/java/com/yoyuzh/files/share/FileShareLinkRepository.java similarity index 96% rename from backend/src/main/java/com/yoyuzh/files/FileShareLinkRepository.java rename to backend/src/main/java/com/yoyuzh/files/share/FileShareLinkRepository.java index e184053..9b67bb8 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileShareLinkRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/share/FileShareLinkRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.share; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; diff --git a/backend/src/main/java/com/yoyuzh/files/ImportSharedFileRequest.java b/backend/src/main/java/com/yoyuzh/files/share/ImportSharedFileRequest.java similarity index 78% rename from backend/src/main/java/com/yoyuzh/files/ImportSharedFileRequest.java rename to backend/src/main/java/com/yoyuzh/files/share/ImportSharedFileRequest.java index 48a9b2e..6b53d7a 100644 --- a/backend/src/main/java/com/yoyuzh/files/ImportSharedFileRequest.java +++ b/backend/src/main/java/com/yoyuzh/files/share/ImportSharedFileRequest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.share; import jakarta.validation.constraints.NotBlank; diff --git a/backend/src/main/java/com/yoyuzh/files/ShareV2Service.java b/backend/src/main/java/com/yoyuzh/files/share/ShareV2Service.java similarity index 87% rename from backend/src/main/java/com/yoyuzh/files/ShareV2Service.java rename to backend/src/main/java/com/yoyuzh/files/share/ShareV2Service.java index 0519cd8..27b759c 100644 --- a/backend/src/main/java/com/yoyuzh/files/ShareV2Service.java +++ b/backend/src/main/java/com/yoyuzh/files/share/ShareV2Service.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.share; import com.yoyuzh.api.v2.ApiV2ErrorCode; import com.yoyuzh.api.v2.ApiV2Exception; @@ -7,10 +7,15 @@ import com.yoyuzh.api.v2.shares.ImportShareV2Request; import com.yoyuzh.api.v2.shares.ShareV2Response; import com.yoyuzh.api.v2.shares.VerifySharePasswordV2Request; import com.yoyuzh.auth.User; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.http.ResponseEntity; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; @@ -85,6 +90,17 @@ public class ShareV2Service { return importedFile; } + @Transactional + public ResponseEntity downloadSharedFile(String token, String password) { + FileShareLink shareLink = getShareLink(token); + ensureShareNotExpired(shareLink); + ensureDownloadAllowed(shareLink); + ensurePasswordAccepted(shareLink, password); + + shareLink.setDownloadCount(shareLink.getDownloadCountOrZero() + 1); + return fileService.download(shareLink.getOwner(), shareLink.getFile().getId()); + } + @Transactional public Page listOwnedShares(User user, Pageable pageable) { return fileShareLinkRepository.findByOwnerIdOrderByCreatedAtDesc(user.getId(), pageable) @@ -114,6 +130,18 @@ public class ShareV2Service { throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "import disabled"); } + ensureQuotaAvailable(shareLink); + } + + private void ensureDownloadAllowed(FileShareLink shareLink) { + if (!shareLink.isAllowDownloadEnabled()) { + throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "download disabled"); + } + + ensureQuotaAvailable(shareLink); + } + + private void ensureQuotaAvailable(FileShareLink shareLink) { Integer maxDownloads = shareLink.getMaxDownloads(); if (maxDownloads != null && shareLink.getDownloadCountOrZero() >= maxDownloads) { throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "share quota exceeded"); diff --git a/backend/src/main/java/com/yoyuzh/files/storage/FileContentStorage.java b/backend/src/main/java/com/yoyuzh/files/storage/FileContentStorage.java index b968625..b45eb06 100644 --- a/backend/src/main/java/com/yoyuzh/files/storage/FileContentStorage.java +++ b/backend/src/main/java/com/yoyuzh/files/storage/FileContentStorage.java @@ -48,6 +48,26 @@ public interface FileContentStorage { void deleteBlob(String objectKey); + default String createMultipartUpload(String objectKey, String contentType) { + throw new UnsupportedOperationException("Multipart upload is not supported by this storage"); + } + + default PreparedUpload prepareMultipartPartUpload(String objectKey, + String uploadId, + int partNumber, + String contentType, + long size) { + throw new UnsupportedOperationException("Multipart upload is not supported by this storage"); + } + + default void completeMultipartUpload(String objectKey, String uploadId, java.util.List parts) { + throw new UnsupportedOperationException("Multipart upload is not supported by this storage"); + } + + default void abortMultipartUpload(String objectKey, String uploadId) { + throw new UnsupportedOperationException("Multipart upload is not supported by this storage"); + } + String createBlobDownloadUrl(String objectKey, String filename); void createDirectory(Long userId, String logicalPath); diff --git a/backend/src/main/java/com/yoyuzh/files/storage/MultipartCompletedPart.java b/backend/src/main/java/com/yoyuzh/files/storage/MultipartCompletedPart.java new file mode 100644 index 0000000..9963fca --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/storage/MultipartCompletedPart.java @@ -0,0 +1,7 @@ +package com.yoyuzh.files.storage; + +public record MultipartCompletedPart( + int partNumber, + String etag +) { +} diff --git a/backend/src/main/java/com/yoyuzh/files/storage/S3FileContentStorage.java b/backend/src/main/java/com/yoyuzh/files/storage/S3FileContentStorage.java index d066e4e..2c99ad5 100644 --- a/backend/src/main/java/com/yoyuzh/files/storage/S3FileContentStorage.java +++ b/backend/src/main/java/com/yoyuzh/files/storage/S3FileContentStorage.java @@ -9,22 +9,31 @@ import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest; import java.io.IOException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -160,6 +169,90 @@ public class S3FileContentStorage implements FileContentStorage { } } + @Override + public String createMultipartUpload(String objectKey, String contentType) { + S3FileRuntimeSession session = sessionProvider.currentSession(); + CreateMultipartUploadRequest.Builder requestBuilder = CreateMultipartUploadRequest.builder() + .bucket(session.bucket()) + .key(normalizeObjectKey(objectKey)); + if (StringUtils.hasText(contentType)) { + requestBuilder.contentType(contentType); + } + try { + return session.s3Client().createMultipartUpload(requestBuilder.build()).uploadId(); + } catch (S3Exception ex) { + throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload init failed"); + } + } + + @Override + public PreparedUpload prepareMultipartPartUpload(String objectKey, + String uploadId, + int partNumber, + String contentType, + long size) { + S3FileRuntimeSession session = sessionProvider.currentSession(); + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(session.bucket()) + .key(normalizeObjectKey(objectKey)) + .uploadId(uploadId) + .partNumber(partNumber) + .contentLength(size) + .build(); + UploadPartPresignRequest presignRequest = UploadPartPresignRequest.builder() + .signatureDuration(Duration.ofSeconds(Math.max(1, properties.getTtlSeconds()))) + .uploadPartRequest(uploadPartRequest) + .build(); + PresignedUploadPartRequest presignedRequest = session.s3Presigner().presignUploadPart(presignRequest); + Map headers = flattenSignedHeaders(presignedRequest.signedHeaders()); + if (StringUtils.hasText(contentType)) { + headers.put("Content-Type", contentType); + } + return new PreparedUpload( + true, + presignedRequest.url().toString(), + resolveUploadMethod(presignedRequest), + headers, + objectKey + ); + } + + @Override + public void completeMultipartUpload(String objectKey, String uploadId, List parts) { + S3FileRuntimeSession session = sessionProvider.currentSession(); + List completedParts = parts.stream() + .sorted(Comparator.comparingInt(MultipartCompletedPart::partNumber)) + .map(part -> CompletedPart.builder() + .partNumber(part.partNumber()) + .eTag(part.etag()) + .build()) + .toList(); + try { + session.s3Client().completeMultipartUpload(CompleteMultipartUploadRequest.builder() + .bucket(session.bucket()) + .key(normalizeObjectKey(objectKey)) + .uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()) + .build()); + } catch (S3Exception ex) { + throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload complete failed"); + } + } + + @Override + public void abortMultipartUpload(String objectKey, String uploadId) { + S3FileRuntimeSession session = sessionProvider.currentSession(); + try { + session.s3Client().abortMultipartUpload(AbortMultipartUploadRequest.builder() + .bucket(session.bucket()) + .key(normalizeObjectKey(objectKey)) + .uploadId(uploadId) + .build()); + } catch (S3Exception ex) { + throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload abort failed"); + } + } + @Override public String createBlobDownloadUrl(String objectKey, String filename) { return createDownloadUrl(sessionProvider.currentSession(), normalizeObjectKey(objectKey), filename); @@ -330,6 +423,13 @@ public class S3FileContentStorage implements FileContentStorage { return presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : "POST"; } + private String resolveUploadMethod(PresignedUploadPartRequest presignedRequest) { + if (presignedRequest.httpRequest() == null) { + return "PUT"; + } + return presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : presignedRequest.httpRequest().method().name(); + } + private Map resolveUploadHeaders(PresignedPutObjectRequest presignedRequest, String contentType) { Map headers = flattenSignedHeaders(presignedRequest.signedHeaders()); if (StringUtils.hasText(contentType)) { diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/ArchiveBackgroundTaskHandler.java b/backend/src/main/java/com/yoyuzh/files/tasks/ArchiveBackgroundTaskHandler.java new file mode 100644 index 0000000..1ba11c7 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/ArchiveBackgroundTaskHandler.java @@ -0,0 +1,164 @@ +package com.yoyuzh.files.tasks; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.auth.User; +import com.yoyuzh.auth.UserRepository; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import jakarta.transaction.Transactional; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.util.LinkedHashMap; +import java.util.Map; + +@Component +@Transactional +public class ArchiveBackgroundTaskHandler implements BackgroundTaskHandler { + + private final StoredFileRepository storedFileRepository; + private final UserRepository userRepository; + private final FileService fileService; + private final ObjectMapper objectMapper; + + public ArchiveBackgroundTaskHandler(StoredFileRepository storedFileRepository, + UserRepository userRepository, + FileService fileService, + ObjectMapper objectMapper) { + this.storedFileRepository = storedFileRepository; + this.userRepository = userRepository; + this.fileService = fileService; + this.objectMapper = objectMapper; + } + + @Override + public boolean supports(BackgroundTaskType type) { + return type == BackgroundTaskType.ARCHIVE; + } + + @Override + public BackgroundTaskHandlerResult handle(BackgroundTask task) { + return handle(task, publicStatePatch -> { + }); + } + + @Override + public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) { + Map state = parseState(task.getPrivateStateJson(), task.getPublicStateJson()); + Long fileId = extractLong(state.get("fileId")); + String outputPath = extractText(state.get("outputPath")); + String outputFilename = extractText(state.get("outputFilename")); + if (fileId == null) { + throw new IllegalStateException("archive task missing fileId"); + } + if (!StringUtils.hasText(outputPath) || !StringUtils.hasText(outputFilename)) { + throw new IllegalStateException("archive task missing output target"); + } + + StoredFile source = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId()) + .orElseThrow(() -> new IllegalStateException("archive task file not found")); + User user = userRepository.findById(task.getUserId()) + .orElseThrow(() -> new IllegalStateException("archive task user not found")); + + FileService.ArchiveSourceSummary summary = fileService.summarizeArchiveSource(source); + progressReporter.report(progressPatch(0, summary.fileCount(), 0, summary.directoryCount())); + byte[] archiveBytes = fileService.buildArchiveBytes(source, progress -> + progressReporter.report(progressPatch( + progress.processedFileCount(), + progress.totalFileCount(), + progress.processedDirectoryCount(), + progress.totalDirectoryCount() + ))); + FileMetadataResponse archivedFile = fileService.importExternalFile( + user, + outputPath, + outputFilename, + "application/zip", + archiveBytes.length, + archiveBytes + ); + + Map publicStatePatch = new LinkedHashMap<>(); + publicStatePatch.put("worker", "archive"); + publicStatePatch.put("archivedFileId", archivedFile.id()); + publicStatePatch.put("archivedFilename", archivedFile.filename()); + publicStatePatch.put("archivedPath", archivedFile.path()); + publicStatePatch.put("archiveSize", archiveBytes.length); + publicStatePatch.putAll(progressPatch( + summary.fileCount(), + summary.fileCount(), + summary.directoryCount(), + summary.directoryCount() + )); + return new BackgroundTaskHandlerResult(publicStatePatch); + } + + private Map progressPatch(int processedFileCount, + int totalFileCount, + int processedDirectoryCount, + int totalDirectoryCount) { + Map patch = new LinkedHashMap<>(); + patch.put("processedFileCount", processedFileCount); + patch.put("totalFileCount", totalFileCount); + patch.put("processedDirectoryCount", processedDirectoryCount); + patch.put("totalDirectoryCount", totalDirectoryCount); + patch.put("progressPercent", calculateProgressPercent( + processedFileCount, + totalFileCount, + processedDirectoryCount, + totalDirectoryCount + )); + return patch; + } + + private int calculateProgressPercent(int processedFileCount, + int totalFileCount, + int processedDirectoryCount, + int totalDirectoryCount) { + int total = Math.max(0, totalFileCount) + Math.max(0, totalDirectoryCount); + int processed = Math.max(0, processedFileCount) + Math.max(0, processedDirectoryCount); + if (total <= 0) { + return 100; + } + return Math.min(100, (int) Math.floor((processed * 100.0d) / total)); + } + + private Map parseState(String privateStateJson, String publicStateJson) { + Map state = new LinkedHashMap<>(parseJsonObject(publicStateJson)); + state.putAll(parseJsonObject(privateStateJson)); + return state; + } + + private Map parseJsonObject(String json) { + if (!StringUtils.hasText(json)) { + return Map.of(); + } + try { + return objectMapper.readValue(json, new TypeReference>() { + }); + } catch (JsonProcessingException ex) { + throw new IllegalStateException("archive task state is invalid", ex); + } + } + + private Long extractLong(Object value) { + if (value instanceof Number number) { + return number.longValue(); + } + if (value instanceof String text && StringUtils.hasText(text)) { + return Long.parseLong(text.trim()); + } + return null; + } + + private String extractText(Object value) { + if (value instanceof String text && StringUtils.hasText(text)) { + return text.trim(); + } + return null; + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/BackgroundTask.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTask.java similarity index 71% rename from backend/src/main/java/com/yoyuzh/files/BackgroundTask.java rename to backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTask.java index 608d9e4..369e290 100644 --- a/backend/src/main/java/com/yoyuzh/files/BackgroundTask.java +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTask.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.tasks; import jakarta.persistence.Column; import jakarta.persistence.Entity; @@ -18,6 +18,7 @@ import java.time.LocalDateTime; @Table(name = "portal_background_task", indexes = { @Index(name = "idx_background_task_user_created_at", columnList = "user_id,created_at"), @Index(name = "idx_background_task_status_created_at", columnList = "status,created_at"), + @Index(name = "idx_background_task_status_lease_expires_at", columnList = "status,lease_expires_at"), @Index(name = "idx_background_task_correlation_id", columnList = "correlation_id") }) public class BackgroundTask { @@ -49,6 +50,24 @@ public class BackgroundTask { @Column(name = "error_message", length = 512) private String errorMessage; + @Column(name = "attempt_count", nullable = false) + private Integer attemptCount; + + @Column(name = "max_attempts", nullable = false) + private Integer maxAttempts; + + @Column(name = "next_run_at") + private LocalDateTime nextRunAt; + + @Column(name = "lease_owner", length = 128) + private String leaseOwner; + + @Column(name = "lease_expires_at") + private LocalDateTime leaseExpiresAt; + + @Column(name = "heartbeat_at") + private LocalDateTime heartbeatAt; + @Column(name = "created_at", nullable = false) private LocalDateTime createdAt; @@ -70,6 +89,12 @@ public class BackgroundTask { if (status == null) { status = BackgroundTaskStatus.QUEUED; } + if (attemptCount == null) { + attemptCount = 0; + } + if (maxAttempts == null) { + maxAttempts = 1; + } if (publicStateJson == null) { publicStateJson = "{}"; } @@ -147,6 +172,54 @@ public class BackgroundTask { this.errorMessage = errorMessage; } + public Integer getAttemptCount() { + return attemptCount; + } + + public void setAttemptCount(Integer attemptCount) { + this.attemptCount = attemptCount; + } + + public Integer getMaxAttempts() { + return maxAttempts; + } + + public void setMaxAttempts(Integer maxAttempts) { + this.maxAttempts = maxAttempts; + } + + public LocalDateTime getNextRunAt() { + return nextRunAt; + } + + public void setNextRunAt(LocalDateTime nextRunAt) { + this.nextRunAt = nextRunAt; + } + + public String getLeaseOwner() { + return leaseOwner; + } + + public void setLeaseOwner(String leaseOwner) { + this.leaseOwner = leaseOwner; + } + + public LocalDateTime getLeaseExpiresAt() { + return leaseExpiresAt; + } + + public void setLeaseExpiresAt(LocalDateTime leaseExpiresAt) { + this.leaseExpiresAt = leaseExpiresAt; + } + + public LocalDateTime getHeartbeatAt() { + return heartbeatAt; + } + + public void setHeartbeatAt(LocalDateTime heartbeatAt) { + this.heartbeatAt = heartbeatAt; + } + public LocalDateTime getCreatedAt() { return createdAt; } diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskFailureCategory.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskFailureCategory.java new file mode 100644 index 0000000..d3fc39d --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskFailureCategory.java @@ -0,0 +1,19 @@ +package com.yoyuzh.files.tasks; + +public enum BackgroundTaskFailureCategory { + UNSUPPORTED_INPUT(false), + DATA_STATE(false), + TRANSIENT_INFRASTRUCTURE(true), + RATE_LIMITED(true), + UNKNOWN(true); + + private final boolean retryable; + + BackgroundTaskFailureCategory(boolean retryable) { + this.retryable = retryable; + } + + public boolean isRetryable() { + return retryable; + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskHandler.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskHandler.java new file mode 100644 index 0000000..4ed07e8 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskHandler.java @@ -0,0 +1,12 @@ +package com.yoyuzh.files.tasks; + +public interface BackgroundTaskHandler { + + boolean supports(BackgroundTaskType type); + + BackgroundTaskHandlerResult handle(BackgroundTask task); + + default BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) { + return handle(task); + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskHandlerResult.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskHandlerResult.java similarity index 87% rename from backend/src/main/java/com/yoyuzh/files/BackgroundTaskHandlerResult.java rename to backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskHandlerResult.java index 3bdeb19..fa37f76 100644 --- a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskHandlerResult.java +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskHandlerResult.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.tasks; import java.util.Map; diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskLeaseLostException.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskLeaseLostException.java new file mode 100644 index 0000000..079708e --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskLeaseLostException.java @@ -0,0 +1,8 @@ +package com.yoyuzh.files.tasks; + +class BackgroundTaskLeaseLostException extends RuntimeException { + + BackgroundTaskLeaseLostException(Long taskId, String workerOwner) { + super("background task lease lost: taskId=" + taskId + ", workerOwner=" + workerOwner); + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskProgressReporter.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskProgressReporter.java new file mode 100644 index 0000000..ff7e995 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskProgressReporter.java @@ -0,0 +1,9 @@ +package com.yoyuzh.files.tasks; + +import java.util.Map; + +@FunctionalInterface +public interface BackgroundTaskProgressReporter { + + void report(Map publicStatePatch); +} diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskRepository.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskRepository.java new file mode 100644 index 0000000..7df4535 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskRepository.java @@ -0,0 +1,103 @@ +package com.yoyuzh.files.tasks; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +public interface BackgroundTaskRepository extends JpaRepository { + + Page findByUserIdOrderByCreatedAtDesc(Long userId, Pageable pageable); + + Optional findByIdAndUserId(Long id, Long userId); + + List findByStatusOrderByCreatedAtAsc(BackgroundTaskStatus status, Pageable pageable); + + List findByStatusOrderByUpdatedAtAsc(BackgroundTaskStatus status); + + @Query(""" + select task.id from BackgroundTask task + where task.status = :status + and (task.nextRunAt is null or task.nextRunAt <= :now) + order by coalesce(task.nextRunAt, task.createdAt) asc, task.createdAt asc + """) + List findReadyTaskIdsByStatusOrder(@Param("status") BackgroundTaskStatus status, + @Param("now") LocalDateTime now, + Pageable pageable); + + @Modifying + @Query(""" + update BackgroundTask task + set task.status = :runningStatus, + task.errorMessage = null, + task.nextRunAt = null, + task.attemptCount = task.attemptCount + 1, + task.leaseOwner = :leaseOwner, + task.leaseExpiresAt = :leaseExpiresAt, + task.heartbeatAt = :heartbeatAt, + task.updatedAt = :updatedAt + where task.id = :id + and task.status = :queuedStatus + """) + int claimQueuedTask(@Param("id") Long id, + @Param("queuedStatus") BackgroundTaskStatus queuedStatus, + @Param("runningStatus") BackgroundTaskStatus runningStatus, + @Param("leaseOwner") String leaseOwner, + @Param("leaseExpiresAt") LocalDateTime leaseExpiresAt, + @Param("heartbeatAt") LocalDateTime heartbeatAt, + @Param("updatedAt") LocalDateTime updatedAt); + + @Query(""" + select task.id from BackgroundTask task + where task.status = :status + and (task.leaseExpiresAt is null or task.leaseExpiresAt <= :now) + order by coalesce(task.leaseExpiresAt, task.updatedAt, task.createdAt) asc + """) + List findExpiredRunningTaskIds(@Param("status") BackgroundTaskStatus status, + @Param("now") LocalDateTime now, + Pageable pageable); + + @Modifying + @Query(""" + update BackgroundTask task + set task.status = :queuedStatus, + task.errorMessage = null, + task.finishedAt = null, + task.nextRunAt = null, + task.leaseOwner = null, + task.leaseExpiresAt = null, + task.heartbeatAt = null, + task.updatedAt = :updatedAt + where task.id = :id + and task.status = :runningStatus + and (task.leaseExpiresAt is null or task.leaseExpiresAt <= :now) + """) + int requeueExpiredRunningTask(@Param("id") Long id, + @Param("runningStatus") BackgroundTaskStatus runningStatus, + @Param("queuedStatus") BackgroundTaskStatus queuedStatus, + @Param("now") LocalDateTime now, + @Param("updatedAt") LocalDateTime updatedAt); + + @Modifying + @Query(""" + update BackgroundTask task + set task.leaseExpiresAt = :leaseExpiresAt, + task.heartbeatAt = :heartbeatAt, + task.updatedAt = :updatedAt + where task.id = :id + and task.status = :runningStatus + and task.leaseOwner = :leaseOwner + """) + int refreshRunningTaskLease(@Param("id") Long id, + @Param("runningStatus") BackgroundTaskStatus runningStatus, + @Param("leaseOwner") String leaseOwner, + @Param("leaseExpiresAt") LocalDateTime leaseExpiresAt, + @Param("heartbeatAt") LocalDateTime heartbeatAt, + @Param("updatedAt") LocalDateTime updatedAt); +} diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskService.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskService.java new file mode 100644 index 0000000..9da562d --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskService.java @@ -0,0 +1,683 @@ +package com.yoyuzh.files.tasks; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.api.v2.ApiV2ErrorCode; +import com.yoyuzh.api.v2.ApiV2Exception; +import com.yoyuzh.auth.User; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import jakarta.transaction.Transactional; +import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import java.time.LocalDateTime; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +@Service +@RequiredArgsConstructor +public class BackgroundTaskService { + + static final String STATE_PHASE_KEY = "phase"; + static final String STATE_ATTEMPT_COUNT_KEY = "attemptCount"; + static final String STATE_MAX_ATTEMPTS_KEY = "maxAttempts"; + static final String STATE_RETRY_SCHEDULED_KEY = "retryScheduled"; + static final String STATE_NEXT_RETRY_AT_KEY = "nextRetryAt"; + static final String STATE_RETRY_DELAY_SECONDS_KEY = "retryDelaySeconds"; + static final String STATE_LAST_FAILURE_MESSAGE_KEY = "lastFailureMessage"; + static final String STATE_LAST_FAILURE_AT_KEY = "lastFailureAt"; + static final String STATE_FAILURE_CATEGORY_KEY = "failureCategory"; + static final String STATE_WORKER_OWNER_KEY = "workerOwner"; + static final String STATE_HEARTBEAT_AT_KEY = "heartbeatAt"; + static final String STATE_LEASE_EXPIRES_AT_KEY = "leaseExpiresAt"; + static final String STATE_STARTED_AT_KEY = "startedAt"; + + private static final List ZIP_COMPATIBLE_EXTENSIONS = List.of(".zip", ".jar", ".war"); + private static final List MEDIA_EXTENSIONS = List.of( + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg", + ".mp4", ".mov", ".mkv", ".webm", ".avi", + ".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a" + ); + private static final List RETRY_TRANSIENT_STATE_KEYS = List.of( + STATE_RETRY_SCHEDULED_KEY, + STATE_NEXT_RETRY_AT_KEY, + STATE_RETRY_DELAY_SECONDS_KEY, + STATE_LAST_FAILURE_MESSAGE_KEY, + STATE_LAST_FAILURE_AT_KEY, + STATE_FAILURE_CATEGORY_KEY + ); + private static final List RUNNING_TRANSIENT_STATE_KEYS = List.of( + STATE_WORKER_OWNER_KEY, + STATE_LEASE_EXPIRES_AT_KEY + ); + private static final int EXPIRED_RUNNING_TASK_BATCH_SIZE = 100; + + private final BackgroundTaskRepository backgroundTaskRepository; + private final StoredFileRepository storedFileRepository; + private final ObjectMapper objectMapper; + + @Transactional + public BackgroundTask createQueuedFileTask(User user, + BackgroundTaskType type, + Long fileId, + String requestedPath, + String correlationId) { + StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, user.getId()) + .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "file not found")); + String logicalPath = buildLogicalPath(file); + if (!logicalPath.equals(normalizeLogicalPath(requestedPath))) { + throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task path does not match file path"); + } + validateTaskTarget(type, file); + + Map publicState = fileState(file, logicalPath); + Map privateState = new LinkedHashMap<>(publicState); + privateState.put("taskType", type.name()); + if (type == BackgroundTaskType.ARCHIVE) { + String outputPath = file.getPath(); + String outputFilename = file.getFilename() + ".zip"; + publicState.put("outputPath", outputPath); + publicState.put("outputFilename", outputFilename); + privateState.put("outputPath", outputPath); + privateState.put("outputFilename", outputFilename); + } else if (type == BackgroundTaskType.EXTRACT) { + String outputPath = file.getPath(); + String outputDirectoryName = deriveExtractOutputDirectoryName(file.getFilename()); + publicState.put("outputPath", outputPath); + publicState.put("outputDirectoryName", outputDirectoryName); + privateState.put("outputPath", outputPath); + privateState.put("outputDirectoryName", outputDirectoryName); + } + return createQueuedTask(user, type, publicState, privateState, correlationId); + } + + @Transactional + public BackgroundTask createQueuedTask(User user, + BackgroundTaskType type, + Map publicState, + Map privateState, + String correlationId) { + BackgroundTask task = new BackgroundTask(); + task.setUserId(user.getId()); + task.setType(type); + task.setStatus(BackgroundTaskStatus.QUEUED); + task.setAttemptCount(0); + task.setMaxAttempts(resolveMaxAttempts(type)); + task.setNextRunAt(null); + Map nextPublicState = new LinkedHashMap<>(publicState == null ? Map.of() : publicState); + nextPublicState.put(STATE_PHASE_KEY, "queued"); + nextPublicState.putAll(retryStatePatch(task.getAttemptCount(), task.getMaxAttempts())); + task.setPublicStateJson(toJson(nextPublicState)); + task.setPrivateStateJson(toJson(privateState)); + task.setCorrelationId(normalizeCorrelationId(correlationId)); + return backgroundTaskRepository.save(task); + } + + public Page listOwnedTasks(User user, Pageable pageable) { + return backgroundTaskRepository.findByUserIdOrderByCreatedAtDesc(user.getId(), pageable); + } + + public BackgroundTask getOwnedTask(User user, Long id) { + return backgroundTaskRepository.findByIdAndUserId(id, user.getId()) + .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found")); + } + + @Transactional + public BackgroundTask cancelOwnedTask(User user, Long id) { + BackgroundTask task = getOwnedTask(user, id); + if (task.isTerminal()) { + return task; + } + + if (task.getStatus() == BackgroundTaskStatus.QUEUED || task.getStatus() == BackgroundTaskStatus.RUNNING) { + task.setStatus(BackgroundTaskStatus.CANCELLED); + task.setNextRunAt(null); + clearLease(task); + task.setPublicStateJson(mergePublicStateJson( + task.getPublicStateJson(), + Map.of( + STATE_PHASE_KEY, "cancelled", + STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(), + STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(), + STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString() + ), + removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS) + )); + task.setFinishedAt(LocalDateTime.now()); + task.setErrorMessage(null); + return backgroundTaskRepository.save(task); + } + + return task; + } + + @Transactional + public BackgroundTask retryOwnedTask(User user, Long id) { + BackgroundTask task = getOwnedTask(user, id); + if (task.getStatus() != BackgroundTaskStatus.FAILED) { + throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "only failed tasks can be retried"); + } + + task.setAttemptCount(0); + task.setNextRunAt(null); + clearLease(task); + task.setPublicStateJson(resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts())); + task.setStatus(BackgroundTaskStatus.QUEUED); + task.setFinishedAt(null); + task.setErrorMessage(null); + return backgroundTaskRepository.save(task); + } + + @Transactional + public BackgroundTask markRunning(User user, Long id) { + BackgroundTask task = getOwnedTask(user, id); + if (task.isTerminal()) { + return task; + } + task.setStatus(BackgroundTaskStatus.RUNNING); + task.setPublicStateJson(mergePublicStateJson( + task.getPublicStateJson(), + Map.of( + STATE_PHASE_KEY, "running", + STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(), + STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts() + ), + List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY) + )); + return backgroundTaskRepository.save(task); + } + + @Transactional + public BackgroundTask markCompleted(User user, Long id) { + BackgroundTask task = getOwnedTask(user, id); + if (task.isTerminal()) { + return task; + } + task.setStatus(BackgroundTaskStatus.COMPLETED); + task.setNextRunAt(null); + clearLease(task); + task.setPublicStateJson(mergePublicStateJson( + task.getPublicStateJson(), + Map.of( + STATE_PHASE_KEY, "completed", + STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(), + STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(), + STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString() + ), + removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS) + )); + task.setFinishedAt(LocalDateTime.now()); + task.setErrorMessage(null); + return backgroundTaskRepository.save(task); + } + + @Transactional + public BackgroundTask markFailed(User user, Long id, String errorMessage) { + BackgroundTask task = getOwnedTask(user, id); + if (task.isTerminal()) { + return task; + } + task.setStatus(BackgroundTaskStatus.FAILED); + task.setNextRunAt(null); + clearLease(task); + task.setPublicStateJson(mergePublicStateJson( + task.getPublicStateJson(), + Map.of( + STATE_PHASE_KEY, "failed", + STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(), + STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(), + STATE_LAST_FAILURE_MESSAGE_KEY, StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed", + STATE_LAST_FAILURE_AT_KEY, LocalDateTime.now().toString(), + STATE_FAILURE_CATEGORY_KEY, BackgroundTaskFailureCategory.UNKNOWN.name(), + STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString() + ), + removableStateKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY), RUNNING_TRANSIENT_STATE_KEYS) + )); + task.setFinishedAt(LocalDateTime.now()); + task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed"); + return backgroundTaskRepository.save(task); + } + + @Transactional + public int requeueExpiredRunningTasks() { + LocalDateTime now = LocalDateTime.now(); + int recovered = 0; + for (Long taskId : backgroundTaskRepository.findExpiredRunningTaskIds( + BackgroundTaskStatus.RUNNING, + now, + PageRequest.of(0, EXPIRED_RUNNING_TASK_BATCH_SIZE) + )) { + int requeued = backgroundTaskRepository.requeueExpiredRunningTask( + taskId, + BackgroundTaskStatus.RUNNING, + BackgroundTaskStatus.QUEUED, + now, + now + ); + if (requeued != 1) { + continue; + } + BackgroundTask task = backgroundTaskRepository.findById(taskId) + .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found")); + resetTaskToQueued(task); + backgroundTaskRepository.save(task); + recovered += 1; + } + return recovered; + } + + public List findQueuedTaskIds(int limit) { + if (limit <= 0) { + return List.of(); + } + + return backgroundTaskRepository.findReadyTaskIdsByStatusOrder( + BackgroundTaskStatus.QUEUED, + LocalDateTime.now(), + PageRequest.of(0, limit) + ); + } + + @Transactional + public Optional claimQueuedTask(Long id, String workerOwner, long leaseDurationSeconds) { + LocalDateTime now = LocalDateTime.now(); + LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds)); + int claimed = backgroundTaskRepository.claimQueuedTask( + id, + BackgroundTaskStatus.QUEUED, + BackgroundTaskStatus.RUNNING, + workerOwner, + leaseExpiresAt, + now, + now + ); + if (claimed != 1) { + return Optional.empty(); + } + Optional task = backgroundTaskRepository.findById(id); + task.ifPresent(claimedTask -> { + claimedTask.setLeaseOwner(workerOwner); + claimedTask.setLeaseExpiresAt(leaseExpiresAt); + claimedTask.setHeartbeatAt(now); + claimedTask.setPublicStateJson(mergePublicStateJson( + claimedTask.getPublicStateJson(), + runningStatePatch(claimedTask, workerOwner, now, leaseExpiresAt, true), + RETRY_TRANSIENT_STATE_KEYS + )); + }); + task.ifPresent(backgroundTaskRepository::save); + return task; + } + + @Transactional + public BackgroundTask markWorkerTaskProgress(Long id, + String workerOwner, + Map publicStatePatch, + long leaseDurationSeconds) { + LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds); + BackgroundTask task = backgroundTaskRepository.findById(id) + .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found")); + task.setLeaseOwner(workerOwner); + task.setLeaseExpiresAt(leaseTouch.leaseExpiresAt()); + task.setHeartbeatAt(leaseTouch.now()); + Map nextPatch = new LinkedHashMap<>(runningStatePatch( + task, + workerOwner, + leaseTouch.now(), + leaseTouch.leaseExpiresAt(), + false + )); + if (publicStatePatch != null) { + nextPatch.putAll(publicStatePatch); + } + task.setPublicStateJson(mergePublicStateJson(task.getPublicStateJson(), nextPatch)); + return backgroundTaskRepository.save(task); + } + + @Transactional + public BackgroundTask markWorkerTaskCompleted(Long id, + String workerOwner, + Map publicStatePatch, + long leaseDurationSeconds) { + LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds); + BackgroundTask task = backgroundTaskRepository.findById(id) + .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found")); + Map nextPatch = new LinkedHashMap<>(publicStatePatch == null ? Map.of() : publicStatePatch); + nextPatch.put(STATE_PHASE_KEY, "completed"); + nextPatch.put(STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount()); + nextPatch.put(STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts()); + nextPatch.put(STATE_HEARTBEAT_AT_KEY, leaseTouch.now().toString()); + task.setPublicStateJson(mergePublicStateJson( + task.getPublicStateJson(), + nextPatch, + removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS) + )); + task.setStatus(BackgroundTaskStatus.COMPLETED); + task.setNextRunAt(null); + clearLease(task); + task.setFinishedAt(LocalDateTime.now()); + task.setErrorMessage(null); + return backgroundTaskRepository.save(task); + } + + @Transactional + public BackgroundTask markWorkerTaskFailed(Long id, + String workerOwner, + String errorMessage, + BackgroundTaskFailureCategory failureCategory, + long leaseDurationSeconds) { + LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds); + BackgroundTask task = backgroundTaskRepository.findById(id) + .orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found")); + String normalizedErrorMessage = StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed"; + LocalDateTime now = leaseTouch.now(); + if (failureCategory.isRetryable() && hasRemainingAttempts(task)) { + long retryDelaySeconds = resolveRetryDelaySeconds(task.getType(), failureCategory, task.getAttemptCount()); + LocalDateTime nextRunAt = now.plusSeconds(retryDelaySeconds); + task.setStatus(BackgroundTaskStatus.QUEUED); + task.setNextRunAt(nextRunAt); + clearLease(task); + task.setFinishedAt(null); + task.setErrorMessage(null); + task.setPublicStateJson(mergePublicStateJson( + task.getPublicStateJson(), + Map.of( + STATE_PHASE_KEY, "queued", + STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(), + STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(), + STATE_RETRY_SCHEDULED_KEY, true, + STATE_NEXT_RETRY_AT_KEY, nextRunAt.toString(), + STATE_RETRY_DELAY_SECONDS_KEY, retryDelaySeconds, + STATE_LAST_FAILURE_MESSAGE_KEY, normalizedErrorMessage, + STATE_LAST_FAILURE_AT_KEY, now.toString(), + STATE_FAILURE_CATEGORY_KEY, failureCategory.name(), + STATE_HEARTBEAT_AT_KEY, now.toString() + ), + RUNNING_TRANSIENT_STATE_KEYS + )); + return backgroundTaskRepository.save(task); + } + + task.setNextRunAt(null); + clearLease(task); + task.setPublicStateJson(mergePublicStateJson( + task.getPublicStateJson(), + Map.of( + STATE_PHASE_KEY, "failed", + STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(), + STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(), + STATE_LAST_FAILURE_MESSAGE_KEY, normalizedErrorMessage, + STATE_LAST_FAILURE_AT_KEY, now.toString(), + STATE_FAILURE_CATEGORY_KEY, failureCategory.name(), + STATE_HEARTBEAT_AT_KEY, now.toString() + ), + removableStateKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY, STATE_RETRY_DELAY_SECONDS_KEY), RUNNING_TRANSIENT_STATE_KEYS) + )); + task.setStatus(BackgroundTaskStatus.FAILED); + task.setFinishedAt(now); + task.setErrorMessage(normalizedErrorMessage); + return backgroundTaskRepository.save(task); + } + + private String normalizeCorrelationId(String correlationId) { + if (StringUtils.hasText(correlationId)) { + return correlationId.trim(); + } + return UUID.randomUUID().toString().replace("-", ""); + } + + private void validateTaskTarget(BackgroundTaskType type, StoredFile file) { + if (type == BackgroundTaskType.ARCHIVE) { + return; + } + if (file.isDirectory()) { + throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task target type is not supported"); + } + if (type == BackgroundTaskType.EXTRACT && !isZipCompatibleArchive(file)) { + throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "extract task only supports zip-compatible archives"); + } + if (type == BackgroundTaskType.MEDIA_META && !isMediaLike(file)) { + throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "media metadata task only supports media files"); + } + } + + private Map fileState(StoredFile file, String logicalPath) { + Map state = new LinkedHashMap<>(); + state.put("fileId", file.getId()); + state.put("path", logicalPath); + state.put("filename", file.getFilename()); + state.put("directory", file.isDirectory()); + state.put("contentType", file.getContentType()); + state.put("size", file.getSize()); + return state; + } + + private boolean isZipCompatibleArchive(StoredFile file) { + String contentType = normalizeContentType(file.getContentType()); + if (contentType.contains("zip") || contentType.contains("java-archive")) { + return true; + } + return hasExtension(file.getFilename(), ZIP_COMPATIBLE_EXTENSIONS); + } + + private boolean isMediaLike(StoredFile file) { + String contentType = normalizeContentType(file.getContentType()); + if (contentType.startsWith("image/") || contentType.startsWith("video/") || contentType.startsWith("audio/")) { + return true; + } + return hasExtension(file.getFilename(), MEDIA_EXTENSIONS); + } + + private String deriveExtractOutputDirectoryName(String filename) { + if (!StringUtils.hasText(filename)) { + return "extracted"; + } + String trimmed = filename.trim(); + String lower = trimmed.toLowerCase(Locale.ROOT); + for (String extension : ZIP_COMPATIBLE_EXTENSIONS) { + if (lower.endsWith(extension) && trimmed.length() > extension.length()) { + return trimmed.substring(0, trimmed.length() - extension.length()); + } + } + int lastDot = trimmed.lastIndexOf('.'); + if (lastDot > 0) { + return trimmed.substring(0, lastDot); + } + return trimmed; + } + + private boolean hasExtension(String filename, List extensions) { + if (!StringUtils.hasText(filename)) { + return false; + } + String normalized = filename.toLowerCase(Locale.ROOT); + return extensions.stream().anyMatch(normalized::endsWith); + } + + private String normalizeContentType(String contentType) { + if (!StringUtils.hasText(contentType)) { + return ""; + } + return contentType.trim().toLowerCase(Locale.ROOT); + } + + private String buildLogicalPath(StoredFile file) { + String parent = normalizeLogicalPath(file.getPath()); + if ("/".equals(parent)) { + return "/" + file.getFilename(); + } + return parent + "/" + file.getFilename(); + } + + private String normalizeLogicalPath(String path) { + if (!StringUtils.hasText(path)) { + return "/"; + } + String normalized = path.trim().replace('\\', '/'); + while (normalized.contains("//")) { + normalized = normalized.replace("//", "/"); + } + if (!normalized.startsWith("/")) { + normalized = "/" + normalized; + } + while (normalized.length() > 1 && normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + private String toJson(Map value) { + Map safeValue = value == null ? new LinkedHashMap<>() : new LinkedHashMap<>(value); + try { + return objectMapper.writeValueAsString(safeValue); + } catch (JsonProcessingException ex) { + throw new IllegalStateException("Failed to serialize background task state", ex); + } + } + + private Map parseJsonObject(String value) { + if (!StringUtils.hasText(value)) { + return new LinkedHashMap<>(); + } + + try { + return objectMapper.readValue(value, new TypeReference>() { + }); + } catch (JsonProcessingException ex) { + throw new IllegalStateException("Failed to parse background task state", ex); + } + } + + private String mergePublicStateJson(String currentValue, Map patch) { + return mergePublicStateJson(currentValue, patch, List.of()); + } + + private String mergePublicStateJson(String currentValue, Map patch, List keysToRemove) { + Map nextPublicState = parseJsonObject(currentValue); + if (keysToRemove != null) { + keysToRemove.forEach(nextPublicState::remove); + } + if (patch != null) { + nextPublicState.putAll(patch); + } + return toJson(nextPublicState); + } + + private String resetPublicStateForRetry(String privateStateJson, int attemptCount, int maxAttempts) { + Map nextPublicState = parseJsonObject(privateStateJson); + nextPublicState.remove("taskType"); + nextPublicState.put(STATE_PHASE_KEY, "queued"); + nextPublicState.putAll(retryStatePatch(attemptCount, maxAttempts)); + return toJson(nextPublicState); + } + + private void resetTaskToQueued(BackgroundTask task) { + task.setNextRunAt(null); + clearLease(task); + task.setPublicStateJson(resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts())); + task.setStatus(BackgroundTaskStatus.QUEUED); + task.setFinishedAt(null); + task.setErrorMessage(null); + } + + private int resolveMaxAttempts(BackgroundTaskType type) { + return switch (type) { + case ARCHIVE -> 4; + case EXTRACT -> 3; + case MEDIA_META -> 2; + default -> 1; + }; + } + + private Map retryStatePatch(int attemptCount, int maxAttempts) { + Map patch = new LinkedHashMap<>(); + patch.put(STATE_ATTEMPT_COUNT_KEY, attemptCount); + patch.put(STATE_MAX_ATTEMPTS_KEY, maxAttempts); + return patch; + } + + private boolean hasRemainingAttempts(BackgroundTask task) { + return task.getAttemptCount() != null + && task.getMaxAttempts() != null + && task.getAttemptCount() < task.getMaxAttempts(); + } + + private long resolveRetryDelaySeconds(BackgroundTaskType type, + BackgroundTaskFailureCategory failureCategory, + Integer attemptCount) { + int safeAttemptCount = attemptCount == null ? 1 : Math.max(1, attemptCount); + long baseDelaySeconds = switch (type) { + case ARCHIVE -> 30L; + case EXTRACT -> 45L; + case MEDIA_META -> 15L; + default -> 30L; + }; + if (failureCategory == BackgroundTaskFailureCategory.RATE_LIMITED) { + baseDelaySeconds *= 4L; + } else if (failureCategory == BackgroundTaskFailureCategory.UNKNOWN) { + baseDelaySeconds *= 2L; + } + long delay = baseDelaySeconds * (1L << Math.min(safeAttemptCount - 1, 2)); + return Math.min(delay, baseDelaySeconds * 4L); + } + + private LeaseTouch refreshLease(Long id, String workerOwner, long leaseDurationSeconds) { + LocalDateTime now = LocalDateTime.now(); + LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds)); + int refreshed = backgroundTaskRepository.refreshRunningTaskLease( + id, + BackgroundTaskStatus.RUNNING, + workerOwner, + leaseExpiresAt, + now, + now + ); + if (refreshed != 1) { + throw new BackgroundTaskLeaseLostException(id, workerOwner); + } + return new LeaseTouch(now, leaseExpiresAt); + } + + private Map runningStatePatch(BackgroundTask task, + String workerOwner, + LocalDateTime heartbeatAt, + LocalDateTime leaseExpiresAt, + boolean includeStartedAt) { + Map patch = new LinkedHashMap<>(); + patch.put(STATE_PHASE_KEY, "running"); + patch.put(STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount()); + patch.put(STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts()); + patch.put(STATE_WORKER_OWNER_KEY, workerOwner); + patch.put(STATE_HEARTBEAT_AT_KEY, heartbeatAt.toString()); + patch.put(STATE_LEASE_EXPIRES_AT_KEY, leaseExpiresAt.toString()); + if (includeStartedAt) { + patch.put(STATE_STARTED_AT_KEY, heartbeatAt.toString()); + } + return patch; + } + + private List removableStateKeys(List primary, List secondary) { + List keys = new java.util.ArrayList<>(primary); + keys.addAll(secondary); + return keys; + } + + private void clearLease(BackgroundTask task) { + task.setLeaseOwner(null); + task.setLeaseExpiresAt(null); + task.setHeartbeatAt(null); + } + + private record LeaseTouch(LocalDateTime now, LocalDateTime leaseExpiresAt) { + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskStartupRecovery.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskStartupRecovery.java new file mode 100644 index 0000000..dbeb9e6 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskStartupRecovery.java @@ -0,0 +1,23 @@ +package com.yoyuzh.files.tasks; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +@Slf4j +public class BackgroundTaskStartupRecovery { + + private final BackgroundTaskService backgroundTaskService; + + @EventListener(ApplicationReadyEvent.class) + public void recoverOnStartup() { + int recovered = backgroundTaskService.requeueExpiredRunningTasks(); + if (recovered > 0) { + log.warn("Recovered {} expired RUNNING background task leases back to QUEUED on startup", recovered); + } + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskStatus.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskStatus.java similarity index 76% rename from backend/src/main/java/com/yoyuzh/files/BackgroundTaskStatus.java rename to backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskStatus.java index 919ed35..8584582 100644 --- a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskStatus.java +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskStatus.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.tasks; public enum BackgroundTaskStatus { QUEUED, diff --git a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskType.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskType.java similarity index 81% rename from backend/src/main/java/com/yoyuzh/files/BackgroundTaskType.java rename to backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskType.java index 65c4983..60c8365 100644 --- a/backend/src/main/java/com/yoyuzh/files/BackgroundTaskType.java +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskType.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.tasks; public enum BackgroundTaskType { ARCHIVE, diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskWorker.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskWorker.java new file mode 100644 index 0000000..05d42f1 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskWorker.java @@ -0,0 +1,167 @@ +package com.yoyuzh.files.tasks; + +import com.yoyuzh.common.BusinessException; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeoutException; + +@Component +public class BackgroundTaskWorker { + + private static final int DEFAULT_BATCH_SIZE = 5; + private static final long DEFAULT_LEASE_DURATION_SECONDS = 120L; + + private final BackgroundTaskService backgroundTaskService; + private final List handlers; + private final String workerOwner; + + public BackgroundTaskWorker(BackgroundTaskService backgroundTaskService, + List handlers) { + this.backgroundTaskService = backgroundTaskService; + this.handlers = List.copyOf(handlers); + this.workerOwner = UUID.randomUUID().toString().replace("-", ""); + } + + @Scheduled( + fixedDelayString = "${app.background-tasks.worker.fixed-delay-ms:30000}", + initialDelayString = "${app.background-tasks.worker.initial-delay-ms:30000}" + ) + public void runScheduledBatch() { + processQueuedTasks(DEFAULT_BATCH_SIZE); + } + + public int processQueuedTasks(int maxTasks) { + backgroundTaskService.requeueExpiredRunningTasks(); + int processedCount = 0; + for (Long taskId : backgroundTaskService.findQueuedTaskIds(maxTasks)) { + var claimedTask = backgroundTaskService.claimQueuedTask(taskId, workerOwner, DEFAULT_LEASE_DURATION_SECONDS); + if (claimedTask.isEmpty()) { + continue; + } + + execute(claimedTask.get()); + processedCount += 1; + } + return processedCount; + } + + private void execute(BackgroundTask task) { + try { + backgroundTaskService.markWorkerTaskProgress( + task.getId(), + workerOwner, + Map.of(BackgroundTaskService.STATE_PHASE_KEY, resolveRunningPhase(task.getType())), + DEFAULT_LEASE_DURATION_SECONDS + ); + BackgroundTaskHandler handler = findHandler(task); + BackgroundTaskHandlerResult result = handler.handle(task, publicStatePatch -> + backgroundTaskService.markWorkerTaskProgress( + task.getId(), + workerOwner, + publicStatePatch, + DEFAULT_LEASE_DURATION_SECONDS + )); + backgroundTaskService.markWorkerTaskCompleted( + task.getId(), + workerOwner, + result.publicStatePatch(), + DEFAULT_LEASE_DURATION_SECONDS + ); + } catch (BackgroundTaskLeaseLostException ignored) { + // Another worker reclaimed the task after this worker stopped heartbeating. + } catch (Exception ex) { + String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage(); + try { + backgroundTaskService.markWorkerTaskFailed( + task.getId(), + workerOwner, + message, + classifyFailure(ex), + DEFAULT_LEASE_DURATION_SECONDS + ); + } catch (BackgroundTaskLeaseLostException ignored) { + // Lease already moved to another worker; keep current worker from overwriting state. + } + } + } + + private String resolveRunningPhase(BackgroundTaskType type) { + return switch (type) { + case ARCHIVE -> "archiving"; + case EXTRACT -> "extracting"; + case MEDIA_META -> "extracting-metadata"; + default -> "running"; + }; + } + + private BackgroundTaskHandler findHandler(BackgroundTask task) { + return handlers.stream() + .filter(handler -> handler.supports(task.getType())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No background task handler for " + task.getType())); + } + + private BackgroundTaskFailureCategory classifyFailure(Throwable throwable) { + if (containsRateLimitSignal(throwable)) { + return BackgroundTaskFailureCategory.RATE_LIMITED; + } + Throwable current = throwable; + while (current != null) { + if (current instanceof BusinessException || current instanceof IllegalArgumentException) { + return BackgroundTaskFailureCategory.UNSUPPORTED_INPUT; + } + if (current instanceof IllegalStateException) { + return BackgroundTaskFailureCategory.DATA_STATE; + } + if (current instanceof SocketTimeoutException + || current instanceof TimeoutException + || current instanceof ConnectException + || current instanceof IOException + || current instanceof UncheckedIOException) { + return BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE; + } + current = current.getCause(); + } + if (containsTransientInfrastructureSignal(throwable)) { + return BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE; + } + return BackgroundTaskFailureCategory.UNKNOWN; + } + + private boolean containsRateLimitSignal(Throwable throwable) { + String message = collectMessages(throwable); + return message.contains("429") + || message.contains("too many requests") + || message.contains("rate limit") + || message.contains("throttle"); + } + + private boolean containsTransientInfrastructureSignal(Throwable throwable) { + String message = collectMessages(throwable); + return message.contains("timeout") + || message.contains("temporarily unavailable") + || message.contains("connection reset") + || message.contains("broken pipe") + || message.contains("connection refused"); + } + + private String collectMessages(Throwable throwable) { + StringBuilder builder = new StringBuilder(); + Throwable current = throwable; + while (current != null) { + if (current.getMessage() != null) { + builder.append(current.getMessage().toLowerCase()).append(' '); + } + current = current.getCause(); + } + return builder.toString(); + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/ExtractBackgroundTaskHandler.java b/backend/src/main/java/com/yoyuzh/files/tasks/ExtractBackgroundTaskHandler.java new file mode 100644 index 0000000..1100d30 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/tasks/ExtractBackgroundTaskHandler.java @@ -0,0 +1,356 @@ +package com.yoyuzh.files.tasks; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.auth.User; +import com.yoyuzh.auth.UserRepository; +import com.yoyuzh.common.BusinessException; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import jakarta.transaction.Transactional; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +@Component +@Transactional +public class ExtractBackgroundTaskHandler implements BackgroundTaskHandler { + + private final StoredFileRepository storedFileRepository; + private final UserRepository userRepository; + private final FileService fileService; + private final ObjectMapper objectMapper; + + public ExtractBackgroundTaskHandler(StoredFileRepository storedFileRepository, + UserRepository userRepository, + FileService fileService, + ObjectMapper objectMapper) { + this.storedFileRepository = storedFileRepository; + this.userRepository = userRepository; + this.fileService = fileService; + this.objectMapper = objectMapper; + } + + @Override + public boolean supports(BackgroundTaskType type) { + return type == BackgroundTaskType.EXTRACT; + } + + @Override + public BackgroundTaskHandlerResult handle(BackgroundTask task) { + return handle(task, publicStatePatch -> { + }); + } + + @Override + public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) { + Map state = parseState(task.getPrivateStateJson(), task.getPublicStateJson()); + Long fileId = extractLong(state.get("fileId")); + String outputPath = extractText(state.get("outputPath")); + String outputDirectoryName = extractText(state.get("outputDirectoryName")); + if (fileId == null) { + throw new IllegalStateException("extract task missing fileId"); + } + if (!StringUtils.hasText(outputPath) || !StringUtils.hasText(outputDirectoryName)) { + throw new IllegalStateException("extract task missing output target"); + } + + StoredFile archive = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId()) + .orElseThrow(() -> new IllegalStateException("extract task file not found")); + if (archive.isDirectory()) { + throw new IllegalStateException("extract task only supports files"); + } + User user = userRepository.findById(task.getUserId()) + .orElseThrow(() -> new IllegalStateException("extract task user not found")); + + ExtractPlan plan = parseArchivePlan(archive, outputPath, outputDirectoryName); + progressReporter.report(progressPatch(0, plan.files().size(), 0, plan.directories().size())); + executePlan(user, plan, progressReporter); + + Map publicStatePatch = new LinkedHashMap<>(); + publicStatePatch.put("worker", "extract"); + publicStatePatch.put("extractedPath", plan.extractedPath()); + publicStatePatch.put("extractedFileCount", plan.files().size()); + publicStatePatch.put("extractedDirectoryCount", plan.directories().size()); + publicStatePatch.putAll(progressPatch( + plan.files().size(), + plan.files().size(), + plan.directories().size(), + plan.directories().size() + )); + return new BackgroundTaskHandlerResult(publicStatePatch); + } + + private void executePlan(User user, ExtractPlan plan, BackgroundTaskProgressReporter progressReporter) { + fileService.importExternalFilesAtomically( + user, + plan.directories(), + plan.files().stream() + .map(file -> new FileService.ExternalFileImport( + file.parentPath(), + file.filename(), + file.contentType(), + file.content() + )) + .toList(), + progress -> progressReporter.report(progressPatch( + progress.processedFileCount(), + progress.totalFileCount(), + progress.processedDirectoryCount(), + progress.totalDirectoryCount() + )) + ); + } + + private Map progressPatch(int processedFileCount, + int totalFileCount, + int processedDirectoryCount, + int totalDirectoryCount) { + Map patch = new LinkedHashMap<>(); + patch.put("processedFileCount", processedFileCount); + patch.put("totalFileCount", totalFileCount); + patch.put("processedDirectoryCount", processedDirectoryCount); + patch.put("totalDirectoryCount", totalDirectoryCount); + patch.put("progressPercent", calculateProgressPercent( + processedFileCount, + totalFileCount, + processedDirectoryCount, + totalDirectoryCount + )); + return patch; + } + + private int calculateProgressPercent(int processedFileCount, + int totalFileCount, + int processedDirectoryCount, + int totalDirectoryCount) { + int total = Math.max(0, totalFileCount) + Math.max(0, totalDirectoryCount); + int processed = Math.max(0, processedFileCount) + Math.max(0, processedDirectoryCount); + if (total <= 0) { + return 100; + } + return Math.min(100, (int) Math.floor((processed * 100.0d) / total)); + } + + private ExtractPlan parseArchivePlan(StoredFile archive, String outputPath, String outputDirectoryName) { + FileService.ZipCompatibleArchive zipArchive; + try { + zipArchive = fileService.readZipCompatibleArchive(archive); + } catch (BusinessException ex) { + throw new IllegalStateException("extract task only supports zip-compatible archives", ex); + } + + List items = zipArchive.entries().stream() + .map(entry -> toZipItem(entry, zipArchive.commonRootDirectoryName())) + .filter(item -> StringUtils.hasText(item.path())) + .toList(); + if (items.isEmpty()) { + throw new IllegalStateException("extract task archive is empty"); + } + + String normalizedOutputPath = normalizeDirectoryPath(outputPath); + if (shouldExtractSingleFileToParent(items, outputDirectoryName)) { + ZipItem fileItem = items.get(0); + return new ExtractPlan( + List.of(), + List.of(new ExtractedFile( + normalizedOutputPath, + outputDirectoryName, + fileItem.contentType(), + fileItem.content() + )), + normalizedOutputPath + ); + } + + String rootPath = joinPath(normalizedOutputPath, outputDirectoryName); + LinkedHashSet directories = new LinkedHashSet<>(); + directories.add(rootPath); + List files = new ArrayList<>(); + for (ZipItem item : items) { + if (item.directory()) { + directories.add(joinPath(rootPath, trimTrailingSlash(item.path()))); + continue; + } + String relativeParent = extractParentPath(item.path()); + String targetParent = StringUtils.hasText(relativeParent) ? joinPath(rootPath, relativeParent) : rootPath; + collectParentDirectories(directories, rootPath, relativeParent); + files.add(new ExtractedFile( + targetParent, + extractLeafName(item.path()), + item.contentType(), + item.content() + )); + } + + return new ExtractPlan(List.copyOf(directories), List.copyOf(files), rootPath); + } + + private void collectParentDirectories(LinkedHashSet directories, String rootPath, String relativeParent) { + if (!StringUtils.hasText(relativeParent)) { + return; + } + String current = ""; + for (String segment : relativeParent.split("/")) { + current = StringUtils.hasText(current) ? current + "/" + segment : segment; + directories.add(joinPath(rootPath, current)); + } + } + + private boolean shouldExtractSingleFileToParent(List items, String outputDirectoryName) { + if (items.size() != 1) { + return false; + } + ZipItem item = items.get(0); + return !item.directory() + && !item.path().contains("/") + && outputDirectoryName.equals(item.path()); + } + + private ZipItem toZipItem(FileService.ZipCompatibleArchiveEntry entry, String commonRootDirectoryName) { + String path = stripCommonRootDirectory(entry.relativePath(), commonRootDirectoryName); + return new ZipItem(path, entry.directory(), entry.content(), guessContentType(path)); + } + + private String stripCommonRootDirectory(String relativePath, String commonRootDirectoryName) { + if (!StringUtils.hasText(relativePath) || !StringUtils.hasText(commonRootDirectoryName)) { + return relativePath; + } + String prefix = commonRootDirectoryName + "/"; + if (relativePath.equals(commonRootDirectoryName)) { + return ""; + } + if (relativePath.startsWith(prefix)) { + return relativePath.substring(prefix.length()); + } + return relativePath; + } + + private Map parseState(String privateStateJson, String publicStateJson) { + Map state = new LinkedHashMap<>(parseJsonObject(publicStateJson)); + state.putAll(parseJsonObject(privateStateJson)); + return state; + } + + private Map parseJsonObject(String json) { + if (!StringUtils.hasText(json)) { + return Map.of(); + } + try { + return objectMapper.readValue(json, new TypeReference>() { + }); + } catch (JsonProcessingException ex) { + throw new IllegalStateException("extract task state is invalid", ex); + } + } + + private Long extractLong(Object value) { + if (value instanceof Number number) { + return number.longValue(); + } + if (value instanceof String text && StringUtils.hasText(text)) { + return Long.parseLong(text.trim()); + } + return null; + } + + private String extractText(Object value) { + if (value instanceof String text && StringUtils.hasText(text)) { + return text.trim(); + } + return null; + } + + private String normalizeDirectoryPath(String path) { + if (!StringUtils.hasText(path)) { + return "/"; + } + String normalized = path.trim().replace('\\', '/'); + while (normalized.contains("//")) { + normalized = normalized.replace("//", "/"); + } + if (!normalized.startsWith("/")) { + normalized = "/" + normalized; + } + while (normalized.length() > 1 && normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + private String joinPath(String parent, String leaf) { + String normalizedParent = normalizeDirectoryPath(parent); + String normalizedLeaf = trimSlashes(leaf); + if (!StringUtils.hasText(normalizedLeaf)) { + return normalizedParent; + } + if ("/".equals(normalizedParent)) { + return "/" + normalizedLeaf; + } + return normalizedParent + "/" + normalizedLeaf; + } + + private String trimSlashes(String value) { + String normalized = value == null ? "" : value.trim().replace('\\', '/'); + while (normalized.startsWith("/")) { + normalized = normalized.substring(1); + } + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + private String trimTrailingSlash(String value) { + if (value == null) { + return ""; + } + String normalized = value; + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + private String extractParentPath(String path) { + int separator = path.lastIndexOf('/'); + if (separator < 0) { + return ""; + } + return path.substring(0, separator); + } + + private String extractLeafName(String path) { + int separator = path.lastIndexOf('/'); + if (separator < 0) { + return path; + } + return path.substring(separator + 1); + } + + private String guessContentType(String entryPath) { + String contentType = URLConnection.guessContentTypeFromName(entryPath); + if (StringUtils.hasText(contentType)) { + return contentType; + } + return "application/octet-stream"; + } + + private record ZipItem(String path, boolean directory, byte[] content, String contentType) { + } + + private record ExtractedFile(String parentPath, String filename, String contentType, byte[] content) { + } + + private record ExtractPlan(List directories, List files, String extractedPath) { + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/MediaMetadataBackgroundTaskHandler.java b/backend/src/main/java/com/yoyuzh/files/tasks/MediaMetadataBackgroundTaskHandler.java similarity index 90% rename from backend/src/main/java/com/yoyuzh/files/MediaMetadataBackgroundTaskHandler.java rename to backend/src/main/java/com/yoyuzh/files/tasks/MediaMetadataBackgroundTaskHandler.java index b44635d..e31c1e9 100644 --- a/backend/src/main/java/com/yoyuzh/files/MediaMetadataBackgroundTaskHandler.java +++ b/backend/src/main/java/com/yoyuzh/files/tasks/MediaMetadataBackgroundTaskHandler.java @@ -1,8 +1,13 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.tasks; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.files.core.FileBlob; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.search.FileMetadata; +import com.yoyuzh.files.search.FileMetadataRepository; import com.yoyuzh.files.storage.FileContentStorage; import jakarta.transaction.Transactional; import org.springframework.stereotype.Component; @@ -48,6 +53,12 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler @Override public BackgroundTaskHandlerResult handle(BackgroundTask task) { + return handle(task, publicStatePatch -> { + }); + } + + @Override + public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) { Long fileId = readFileId(task); StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId()) .orElseThrow(() -> new IllegalStateException("media metadata task file not found")); @@ -62,6 +73,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler String contentType = firstText(file.getContentType(), blob.getContentType()); long size = firstLong(file.getSize(), blob.getSize()); + progressReporter.report(Map.of("metadataStage", "loading-content")); byte[] content = Optional.ofNullable(fileContentStorage.readBlob(blob.getObjectKey())) .orElseThrow(() -> new IllegalStateException("media metadata task requires blob content")); @@ -75,6 +87,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler upsertMetadata(file, MEDIA_SIZE, String.valueOf(size)); try { + progressReporter.report(Map.of("metadataStage", "reading-image")); BufferedImage image = ImageIO.read(new ByteArrayInputStream(content)); if (image != null) { upsertMetadata(file, MEDIA_WIDTH, String.valueOf(image.getWidth())); @@ -86,6 +99,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler throw new IllegalStateException("media metadata task failed to read image dimensions", ex); } + publicStatePatch.put("metadataStage", "completed"); return new BackgroundTaskHandlerResult(publicStatePatch); } diff --git a/backend/src/main/java/com/yoyuzh/files/NoopBackgroundTaskHandler.java b/backend/src/main/java/com/yoyuzh/files/tasks/NoopBackgroundTaskHandler.java similarity index 85% rename from backend/src/main/java/com/yoyuzh/files/NoopBackgroundTaskHandler.java rename to backend/src/main/java/com/yoyuzh/files/tasks/NoopBackgroundTaskHandler.java index d4f0aa2..98b9a4c 100644 --- a/backend/src/main/java/com/yoyuzh/files/NoopBackgroundTaskHandler.java +++ b/backend/src/main/java/com/yoyuzh/files/tasks/NoopBackgroundTaskHandler.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.tasks; import org.springframework.stereotype.Component; @@ -9,10 +9,7 @@ import java.util.Set; @Component public class NoopBackgroundTaskHandler implements BackgroundTaskHandler { - private static final Set SUPPORTED_TYPES = Set.of( - BackgroundTaskType.ARCHIVE, - BackgroundTaskType.EXTRACT - ); + private static final Set SUPPORTED_TYPES = Set.of(); @Override public boolean supports(BackgroundTaskType type) { diff --git a/backend/src/main/java/com/yoyuzh/files/CompleteUploadRequest.java b/backend/src/main/java/com/yoyuzh/files/upload/CompleteUploadRequest.java similarity index 89% rename from backend/src/main/java/com/yoyuzh/files/CompleteUploadRequest.java rename to backend/src/main/java/com/yoyuzh/files/upload/CompleteUploadRequest.java index 5a53bed..ba851c9 100644 --- a/backend/src/main/java/com/yoyuzh/files/CompleteUploadRequest.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/CompleteUploadRequest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; diff --git a/backend/src/main/java/com/yoyuzh/files/InitiateUploadRequest.java b/backend/src/main/java/com/yoyuzh/files/upload/InitiateUploadRequest.java similarity index 88% rename from backend/src/main/java/com/yoyuzh/files/InitiateUploadRequest.java rename to backend/src/main/java/com/yoyuzh/files/upload/InitiateUploadRequest.java index 10b0bf3..07b0915 100644 --- a/backend/src/main/java/com/yoyuzh/files/InitiateUploadRequest.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/InitiateUploadRequest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; diff --git a/backend/src/main/java/com/yoyuzh/files/InitiateUploadResponse.java b/backend/src/main/java/com/yoyuzh/files/upload/InitiateUploadResponse.java similarity index 86% rename from backend/src/main/java/com/yoyuzh/files/InitiateUploadResponse.java rename to backend/src/main/java/com/yoyuzh/files/upload/InitiateUploadResponse.java index eb8af44..3d7b4df 100644 --- a/backend/src/main/java/com/yoyuzh/files/InitiateUploadResponse.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/InitiateUploadResponse.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; import java.util.Map; diff --git a/backend/src/main/java/com/yoyuzh/files/UploadSession.java b/backend/src/main/java/com/yoyuzh/files/upload/UploadSession.java similarity index 94% rename from backend/src/main/java/com/yoyuzh/files/UploadSession.java rename to backend/src/main/java/com/yoyuzh/files/upload/UploadSession.java index 47214ab..9c54e7e 100644 --- a/backend/src/main/java/com/yoyuzh/files/UploadSession.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/UploadSession.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; import com.yoyuzh.auth.User; import jakarta.persistence.Column; @@ -55,6 +55,9 @@ public class UploadSession { @Column(name = "object_key", nullable = false, length = 512) private String objectKey; + @Column(name = "multipart_upload_id", length = 255) + private String multipartUploadId; + @Column(name = "storage_policy_id") private Long storagePolicyId; @@ -163,6 +166,14 @@ public class UploadSession { this.objectKey = objectKey; } + public String getMultipartUploadId() { + return multipartUploadId; + } + + public void setMultipartUploadId(String multipartUploadId) { + this.multipartUploadId = multipartUploadId; + } + public Long getStoragePolicyId() { return storagePolicyId; } diff --git a/backend/src/main/java/com/yoyuzh/files/UploadSessionCreateCommand.java b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionCreateCommand.java similarity index 81% rename from backend/src/main/java/com/yoyuzh/files/UploadSessionCreateCommand.java rename to backend/src/main/java/com/yoyuzh/files/upload/UploadSessionCreateCommand.java index 952d9fc..a0be788 100644 --- a/backend/src/main/java/com/yoyuzh/files/UploadSessionCreateCommand.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionCreateCommand.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; public record UploadSessionCreateCommand( String path, diff --git a/backend/src/main/java/com/yoyuzh/files/UploadSessionPartCommand.java b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionPartCommand.java similarity index 72% rename from backend/src/main/java/com/yoyuzh/files/UploadSessionPartCommand.java rename to backend/src/main/java/com/yoyuzh/files/upload/UploadSessionPartCommand.java index 51c4b73..fd46d3a 100644 --- a/backend/src/main/java/com/yoyuzh/files/UploadSessionPartCommand.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionPartCommand.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; public record UploadSessionPartCommand( String etag, diff --git a/backend/src/main/java/com/yoyuzh/files/UploadSessionRepository.java b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionRepository.java similarity index 93% rename from backend/src/main/java/com/yoyuzh/files/UploadSessionRepository.java rename to backend/src/main/java/com/yoyuzh/files/upload/UploadSessionRepository.java index 2f29740..459f8d6 100644 --- a/backend/src/main/java/com/yoyuzh/files/UploadSessionRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionRepository.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/backend/src/main/java/com/yoyuzh/files/UploadSessionService.java b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionService.java similarity index 76% rename from backend/src/main/java/com/yoyuzh/files/UploadSessionService.java rename to backend/src/main/java/com/yoyuzh/files/upload/UploadSessionService.java index 560335f..ecd86e3 100644 --- a/backend/src/main/java/com/yoyuzh/files/UploadSessionService.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionService.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -6,7 +6,13 @@ import com.yoyuzh.auth.User; import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.ErrorCode; import com.yoyuzh.config.FileStorageProperties; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.policy.StoragePolicy; +import com.yoyuzh.files.policy.StoragePolicyService; import com.yoyuzh.files.storage.FileContentStorage; +import com.yoyuzh.files.storage.MultipartCompletedPart; +import com.yoyuzh.files.storage.PreparedUpload; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -82,7 +88,8 @@ public class UploadSessionService { session.setContentType(command.contentType()); session.setSize(command.size()); session.setObjectKey(createBlobObjectKey()); - session.setStoragePolicyId(storagePolicyService.ensureDefaultPolicy().getId()); + StoragePolicy policy = storagePolicyService.ensureDefaultPolicy(); + session.setStoragePolicyId(policy.getId()); session.setChunkSize(DEFAULT_CHUNK_SIZE); session.setChunkCount(calculateChunkCount(command.size(), DEFAULT_CHUNK_SIZE)); session.setUploadedPartsJson("[]"); @@ -91,6 +98,9 @@ public class UploadSessionService { session.setCreatedAt(now); session.setUpdatedAt(now); session.setExpiresAt(now.plusHours(SESSION_TTL_HOURS)); + if (storagePolicyService.readCapabilities(policy).multipartUpload()) { + session.setMultipartUploadId(fileContentStorage.createMultipartUpload(session.getObjectKey(), session.getContentType())); + } return uploadSessionRepository.save(session); } @@ -111,6 +121,26 @@ public class UploadSessionService { return uploadSessionRepository.save(session); } + @Transactional(readOnly = true) + public PreparedUpload prepareOwnedPartUpload(User user, String sessionId, int partIndex) { + UploadSession session = getOwnedSession(user, sessionId); + LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); + ensureSessionCanReceivePart(session, now); + if (!StringUtils.hasText(session.getMultipartUploadId())) { + throw new BusinessException(ErrorCode.UNKNOWN, "上传会话未启用 multipart"); + } + if (partIndex < 0 || partIndex >= session.getChunkCount()) { + throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法"); + } + return fileContentStorage.prepareMultipartPartUpload( + session.getObjectKey(), + session.getMultipartUploadId(), + partIndex + 1, + session.getContentType(), + resolveChunkSize(session, partIndex) + ); + } + @Transactional public UploadSession recordUploadedPart(User user, String sessionId, @@ -164,6 +194,13 @@ public class UploadSessionService { uploadSessionRepository.save(session); try { + if (StringUtils.hasText(session.getMultipartUploadId())) { + fileContentStorage.completeMultipartUpload( + session.getObjectKey(), + session.getMultipartUploadId(), + toCompletedParts(session) + ); + } fileService.completeUpload(user, new CompleteUploadRequest( session.getTargetPath(), session.getFilename(), @@ -192,7 +229,11 @@ public class UploadSessionService { ); for (UploadSession session : expiredSessions) { try { - fileContentStorage.deleteBlob(session.getObjectKey()); + if (StringUtils.hasText(session.getMultipartUploadId())) { + fileContentStorage.abortMultipartUpload(session.getObjectKey(), session.getMultipartUploadId()); + } else { + fileContentStorage.deleteBlob(session.getObjectKey()); + } } catch (RuntimeException ignored) { // Expiration is authoritative in the database even if remote object cleanup fails. } @@ -253,6 +294,41 @@ public class UploadSessionService { } } + private List toCompletedParts(UploadSession session) { + List uploadedParts = readUploadedParts(session).stream() + .sorted(Comparator.comparingInt(UploadedPart::partIndex)) + .toList(); + if (uploadedParts.size() != session.getChunkCount()) { + throw new BusinessException(ErrorCode.UNKNOWN, "上传分片不完整"); + } + for (int expectedIndex = 0; expectedIndex < session.getChunkCount(); expectedIndex++) { + UploadedPart part = uploadedParts.get(expectedIndex); + if (part.partIndex() != expectedIndex) { + throw new BusinessException(ErrorCode.UNKNOWN, "上传分片不完整"); + } + if (!StringUtils.hasText(part.etag())) { + throw new BusinessException(ErrorCode.UNKNOWN, "上传分片标识缺失"); + } + if (part.size() <= 0 || part.size() > resolveChunkSize(session, expectedIndex)) { + throw new BusinessException(ErrorCode.UNKNOWN, "上传分片大小不合法"); + } + } + return uploadedParts.stream() + .map(part -> new MultipartCompletedPart(part.partIndex() + 1, part.etag())) + .toList(); + } + + private long resolveChunkSize(UploadSession session, int partIndex) { + if (partIndex < 0 || partIndex >= session.getChunkCount()) { + throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法"); + } + if (partIndex < session.getChunkCount() - 1) { + return session.getChunkSize(); + } + long remaining = session.getSize() - session.getChunkSize() * (session.getChunkCount() - 1L); + return remaining > 0 ? remaining : session.getChunkSize(); + } + private record UploadedPart(int partIndex, String etag, long size, String uploadedAt) { } diff --git a/backend/src/main/java/com/yoyuzh/files/UploadSessionStatus.java b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionStatus.java similarity index 80% rename from backend/src/main/java/com/yoyuzh/files/UploadSessionStatus.java rename to backend/src/main/java/com/yoyuzh/files/upload/UploadSessionStatus.java index 98692f5..8f2d663 100644 --- a/backend/src/main/java/com/yoyuzh/files/UploadSessionStatus.java +++ b/backend/src/main/java/com/yoyuzh/files/upload/UploadSessionStatus.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; public enum UploadSessionStatus { CREATED, diff --git a/backend/src/main/java/com/yoyuzh/transfer/TransferController.java b/backend/src/main/java/com/yoyuzh/transfer/TransferController.java index 15f2a36..629be8a 100644 --- a/backend/src/main/java/com/yoyuzh/transfer/TransferController.java +++ b/backend/src/main/java/com/yoyuzh/transfer/TransferController.java @@ -5,8 +5,8 @@ import com.yoyuzh.auth.User; import com.yoyuzh.common.ApiResponse; import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.ErrorCode; -import com.yoyuzh.files.FileMetadataResponse; -import com.yoyuzh.files.ImportSharedFileRequest; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.share.ImportSharedFileRequest; import io.swagger.v3.oas.annotations.Operation; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/yoyuzh/transfer/TransferService.java b/backend/src/main/java/com/yoyuzh/transfer/TransferService.java index 19cf282..5b3f3e4 100644 --- a/backend/src/main/java/com/yoyuzh/transfer/TransferService.java +++ b/backend/src/main/java/com/yoyuzh/transfer/TransferService.java @@ -5,8 +5,8 @@ import com.yoyuzh.auth.User; import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.ErrorCode; import com.yoyuzh.config.FileStorageProperties; -import com.yoyuzh.files.FileMetadataResponse; -import com.yoyuzh.files.FileService; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.core.FileService; import com.yoyuzh.files.storage.FileContentStorage; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; diff --git a/backend/src/test/java/com/yoyuzh/admin/AdminControllerIntegrationTest.java b/backend/src/test/java/com/yoyuzh/admin/AdminControllerIntegrationTest.java index 3d3ea3e..358611d 100644 --- a/backend/src/test/java/com/yoyuzh/admin/AdminControllerIntegrationTest.java +++ b/backend/src/test/java/com/yoyuzh/admin/AdminControllerIntegrationTest.java @@ -4,10 +4,10 @@ import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.admin.AdminMetricsStateRepository; import com.yoyuzh.auth.User; import com.yoyuzh.auth.UserRepository; -import com.yoyuzh.files.FileBlob; -import com.yoyuzh.files.FileBlobRepository; -import com.yoyuzh.files.StoredFile; -import com.yoyuzh.files.StoredFileRepository; +import com.yoyuzh.files.core.FileBlob; +import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; import com.yoyuzh.transfer.OfflineTransferSessionRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/backend/src/test/java/com/yoyuzh/admin/AdminServiceTest.java b/backend/src/test/java/com/yoyuzh/admin/AdminServiceTest.java index 5aaa352..d8561d7 100644 --- a/backend/src/test/java/com/yoyuzh/admin/AdminServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/admin/AdminServiceTest.java @@ -8,12 +8,12 @@ import com.yoyuzh.auth.UserRepository; import com.yoyuzh.auth.UserRole; import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.PageResponse; -import com.yoyuzh.files.FileBlobRepository; -import com.yoyuzh.files.FileService; -import com.yoyuzh.files.StoredFile; -import com.yoyuzh.files.StoredFileRepository; -import com.yoyuzh.files.StoragePolicyRepository; -import com.yoyuzh.files.StoragePolicyService; +import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.policy.StoragePolicyRepository; +import com.yoyuzh.files.policy.StoragePolicyService; import com.yoyuzh.transfer.OfflineTransferSessionRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/backend/src/test/java/com/yoyuzh/api/v2/files/FileSearchV2ControllerTest.java b/backend/src/test/java/com/yoyuzh/api/v2/files/FileSearchV2ControllerTest.java index 3eb3635..9ff0b8d 100644 --- a/backend/src/test/java/com/yoyuzh/api/v2/files/FileSearchV2ControllerTest.java +++ b/backend/src/test/java/com/yoyuzh/api/v2/files/FileSearchV2ControllerTest.java @@ -4,9 +4,9 @@ import com.yoyuzh.api.v2.ApiV2ExceptionHandler; import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.User; import com.yoyuzh.common.PageResponse; -import com.yoyuzh.files.FileMetadataResponse; -import com.yoyuzh.files.FileSearchQuery; -import com.yoyuzh.files.FileSearchService; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.search.FileSearchQuery; +import com.yoyuzh.files.search.FileSearchService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.core.MethodParameter; diff --git a/backend/src/test/java/com/yoyuzh/api/v2/files/UploadSessionV2ControllerTest.java b/backend/src/test/java/com/yoyuzh/api/v2/files/UploadSessionV2ControllerTest.java index 59bf251..ed46490 100644 --- a/backend/src/test/java/com/yoyuzh/api/v2/files/UploadSessionV2ControllerTest.java +++ b/backend/src/test/java/com/yoyuzh/api/v2/files/UploadSessionV2ControllerTest.java @@ -2,9 +2,9 @@ package com.yoyuzh.api.v2.files; import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.User; -import com.yoyuzh.files.UploadSession; -import com.yoyuzh.files.UploadSessionService; -import com.yoyuzh.files.UploadSessionStatus; +import com.yoyuzh.files.upload.UploadSession; +import com.yoyuzh.files.upload.UploadSessionService; +import com.yoyuzh.files.upload.UploadSessionStatus; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; @@ -19,6 +19,7 @@ import org.springframework.web.method.support.HandlerMethodArgumentResolver; import org.springframework.web.method.support.ModelAndViewContainer; import java.time.LocalDateTime; +import java.util.Map; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -69,6 +70,7 @@ class UploadSessionV2ControllerTest { .andExpect(jsonPath("$.data.sessionId").value("session-1")) .andExpect(jsonPath("$.data.objectKey").value("blobs/session-1")) .andExpect(jsonPath("$.data.status").value("CREATED")) + .andExpect(jsonPath("$.data.multipartUpload").value(true)) .andExpect(jsonPath("$.data.chunkSize").value(8388608)) .andExpect(jsonPath("$.data.chunkCount").value(3)); } @@ -85,7 +87,8 @@ class UploadSessionV2ControllerTest { .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value(0)) .andExpect(jsonPath("$.data.sessionId").value("session-1")) - .andExpect(jsonPath("$.data.status").value("CREATED")); + .andExpect(jsonPath("$.data.status").value("CREATED")) + .andExpect(jsonPath("$.data.multipartUpload").value(true)); } @Test @@ -127,6 +130,29 @@ class UploadSessionV2ControllerTest { .andExpect(jsonPath("$.data.status").value("UPLOADING")); } + @Test + void shouldPrepareMultipartPartUploadWithV2Envelope() throws Exception { + User user = createUser(7L); + when(userDetailsService.loadDomainUser("alice")).thenReturn(user); + when(uploadSessionService.prepareOwnedPartUpload(user, "session-1", 1)) + .thenReturn(new com.yoyuzh.files.storage.PreparedUpload( + true, + "https://upload.example.com/session-1/part-2", + "PUT", + Map.of("Content-Type", "video/mp4"), + "blobs/session-1" + )); + + mockMvc.perform(get("/api/v2/files/upload-sessions/session-1/parts/1/prepare") + .with(user(userDetails()))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.direct").value(true)) + .andExpect(jsonPath("$.data.uploadUrl").value("https://upload.example.com/session-1/part-2")) + .andExpect(jsonPath("$.data.method").value("PUT")) + .andExpect(jsonPath("$.data.headers['Content-Type']").value("video/mp4")); + } + private UserDetails userDetails() { return org.springframework.security.core.userdetails.User .withUsername("alice") @@ -172,6 +198,7 @@ class UploadSessionV2ControllerTest { session.setContentType("video/mp4"); session.setSize(20L * 1024 * 1024); session.setObjectKey("blobs/session-1"); + session.setMultipartUploadId("upload-123"); session.setChunkSize(8L * 1024 * 1024); session.setChunkCount(3); session.setStatus(UploadSessionStatus.CREATED); diff --git a/backend/src/test/java/com/yoyuzh/api/v2/shares/ShareV2ControllerIntegrationTest.java b/backend/src/test/java/com/yoyuzh/api/v2/shares/ShareV2ControllerIntegrationTest.java index c0d44d2..65b6ee9 100644 --- a/backend/src/test/java/com/yoyuzh/api/v2/shares/ShareV2ControllerIntegrationTest.java +++ b/backend/src/test/java/com/yoyuzh/api/v2/shares/ShareV2ControllerIntegrationTest.java @@ -4,12 +4,12 @@ import com.jayway.jsonpath.JsonPath; import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.auth.User; import com.yoyuzh.auth.UserRepository; -import com.yoyuzh.files.FileBlob; -import com.yoyuzh.files.FileBlobRepository; -import com.yoyuzh.files.FileShareLink; -import com.yoyuzh.files.FileShareLinkRepository; -import com.yoyuzh.files.StoredFile; -import com.yoyuzh.files.StoredFileRepository; +import com.yoyuzh.files.core.FileBlob; +import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.files.share.FileShareLink; +import com.yoyuzh.files.share.FileShareLinkRepository; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -30,6 +30,8 @@ import static org.springframework.security.test.web.servlet.request.SecurityMock import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -274,6 +276,120 @@ class ShareV2ControllerIntegrationTest { .andExpect(jsonPath("$.code").value(2404)); } + @Test + void shouldDownloadSharedFileAndCountQuota() throws Exception { + String createResponse = mockMvc.perform(post("/api/v2/shares") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "maxDownloads": 1, + "allowDownload": true + } + """.formatted(sharedFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + String token = JsonPath.read(createResponse, "$.data.token"); + + mockMvc.perform(get("/api/v2/shares/{token}", token) + .with(anonymous()) + .param("download", "1")) + .andExpect(status().isOk()) + .andExpect(header().string("Content-Disposition", "attachment; filename*=UTF-8''notes.txt")) + .andExpect(content().contentType("text/plain")) + .andExpect(content().bytes("hello".getBytes(StandardCharsets.UTF_8))); + + mockMvc.perform(get("/api/v2/shares/{token}", token) + .with(anonymous()) + .param("download", "1")) + .andExpect(status().isForbidden()) + .andExpect(jsonPath("$.code").value(2403)); + + mockMvc.perform(get("/api/v2/shares/{token}", token).with(anonymous())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.downloadCount").value(1)); + } + + @Test + void shouldRejectDownloadWhenSharePolicyBlocksIt() throws Exception { + String passwordResponse = mockMvc.perform(post("/api/v2/shares") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "password": "Share123", + "allowDownload": true + } + """.formatted(sharedFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + String passwordToken = JsonPath.read(passwordResponse, "$.data.token"); + + mockMvc.perform(get("/api/v2/shares/{token}", passwordToken) + .with(anonymous()) + .param("download", "1")) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.code").value(2400)); + + mockMvc.perform(get("/api/v2/shares/{token}", passwordToken) + .with(anonymous()) + .param("download", "1") + .param("password", "Share123")) + .andExpect(status().isOk()) + .andExpect(content().bytes("hello".getBytes(StandardCharsets.UTF_8))); + + String disabledResponse = mockMvc.perform(post("/api/v2/shares") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "allowDownload": false + } + """.formatted(sharedFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + String disabledToken = JsonPath.read(disabledResponse, "$.data.token"); + + mockMvc.perform(get("/api/v2/shares/{token}", disabledToken) + .with(anonymous()) + .param("download", "1")) + .andExpect(status().isForbidden()) + .andExpect(jsonPath("$.code").value(2403)); + + String expiredResponse = mockMvc.perform(post("/api/v2/shares") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "allowDownload": true + } + """.formatted(sharedFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + String expiredToken = JsonPath.read(expiredResponse, "$.data.token"); + FileShareLink expiredShare = fileShareLinkRepository.findByToken(expiredToken).orElseThrow(); + expiredShare.setExpiresAt(LocalDateTime.now().minusMinutes(1)); + fileShareLinkRepository.save(expiredShare); + + mockMvc.perform(get("/api/v2/shares/{token}", expiredToken) + .with(anonymous()) + .param("download", "1")) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.code").value(2404)); + } + @Test void shouldDenyDeletingOtherUsersShare() throws Exception { String createResponse = mockMvc.perform(post("/api/v2/shares") diff --git a/backend/src/test/java/com/yoyuzh/api/v2/tasks/BackgroundTaskV2ControllerIntegrationTest.java b/backend/src/test/java/com/yoyuzh/api/v2/tasks/BackgroundTaskV2ControllerIntegrationTest.java index fb1ce61..603889e 100644 --- a/backend/src/test/java/com/yoyuzh/api/v2/tasks/BackgroundTaskV2ControllerIntegrationTest.java +++ b/backend/src/test/java/com/yoyuzh/api/v2/tasks/BackgroundTaskV2ControllerIntegrationTest.java @@ -1,16 +1,22 @@ package com.yoyuzh.api.v2.tasks; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.JsonPath; import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.auth.User; import com.yoyuzh.auth.UserRepository; -import com.yoyuzh.files.BackgroundTask; -import com.yoyuzh.files.BackgroundTaskRepository; -import com.yoyuzh.files.BackgroundTaskStatus; -import com.yoyuzh.files.BackgroundTaskType; -import com.yoyuzh.files.StoredFile; -import com.yoyuzh.files.StoredFileRepository; +import com.yoyuzh.files.tasks.BackgroundTask; +import com.yoyuzh.files.tasks.BackgroundTaskRepository; +import com.yoyuzh.files.tasks.BackgroundTaskStatus; +import com.yoyuzh.files.tasks.BackgroundTaskStartupRecovery; +import com.yoyuzh.files.tasks.BackgroundTaskType; +import com.yoyuzh.files.tasks.BackgroundTaskWorker; +import com.yoyuzh.files.core.FileBlob; +import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.storage.FileContentStorage; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -19,10 +25,17 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.anonymous; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; @@ -55,23 +68,39 @@ class BackgroundTaskV2ControllerIntegrationTest { @Autowired private BackgroundTaskRepository backgroundTaskRepository; + @Autowired + private BackgroundTaskWorker backgroundTaskWorker; + + @Autowired + private BackgroundTaskStartupRecovery backgroundTaskStartupRecovery; + + @Autowired + private FileBlobRepository fileBlobRepository; + + @Autowired + private FileContentStorage fileContentStorage; + @Autowired private StoredFileRepository storedFileRepository; @Autowired private ObjectMapper objectMapper; + private Long aliceId; private Long archiveDirectoryId; private Long archiveFileId; private Long extractFileId; + private Long invalidExtractFileId; + private Long unsupportedExtractFileId; private Long mediaFileId; private Long foreignFileId; private Long deletedFileId; @BeforeEach - void setUp() { + void setUp() throws Exception { backgroundTaskRepository.deleteAll(); storedFileRepository.deleteAll(); + fileBlobRepository.deleteAll(); userRepository.deleteAll(); User alice = new User(); @@ -80,7 +109,8 @@ class BackgroundTaskV2ControllerIntegrationTest { alice.setPhoneNumber("13800138000"); alice.setPasswordHash("encoded-password"); alice.setCreatedAt(LocalDateTime.now()); - userRepository.save(alice); + alice = userRepository.save(alice); + aliceId = alice.getId(); User bob = new User(); bob.setUsername("bob"); @@ -91,11 +121,62 @@ class BackgroundTaskV2ControllerIntegrationTest { bob = userRepository.save(bob); archiveDirectoryId = storedFileRepository.save(createFile(alice, "/docs", "archive", true, null, 0L, null)).getId(); - archiveFileId = storedFileRepository.save(createFile(alice, "/docs", "archive-source.txt", false, "text/plain", 12L, null)).getId(); - extractFileId = storedFileRepository.save(createFile(alice, "/docs", "extract.zip", false, "application/zip", 32L, null)).getId(); + storedFileRepository.save(createBlobBackedFile( + alice, + "/docs/archive", + "nested.txt", + "text/plain", + "archive-nested", + "nested-content".getBytes(StandardCharsets.UTF_8) + )); + archiveFileId = storedFileRepository.save(createBlobBackedFile( + alice, + "/docs", + "archive-source.txt", + "text/plain", + "archive-source", + "archive-source".getBytes(StandardCharsets.UTF_8) + )).getId(); + extractFileId = storedFileRepository.save(createBlobBackedFile( + alice, + "/docs", + "extract.zip", + "application/zip", + "extract-source", + createZipArchive(Map.of( + "extract/", "", + "extract/nested/", "", + "extract/notes.txt", "hello", + "extract/nested/todo.txt", "world" + )) + )).getId(); + invalidExtractFileId = storedFileRepository.save(createBlobBackedFile( + alice, + "/docs", + "broken.zip", + "application/zip", + "broken-extract", + "not-a-zip".getBytes(StandardCharsets.UTF_8) + )).getId(); + unsupportedExtractFileId = storedFileRepository.save(createFile(alice, "/docs", "backup.7z", false, "application/x-7z-compressed", 64L, null)).getId(); mediaFileId = storedFileRepository.save(createFile(alice, "/docs", "media.png", false, "image/png", 24L, null)).getId(); - foreignFileId = storedFileRepository.save(createFile(bob, "/docs", "foreign.zip", false, "application/zip", 32L, null)).getId(); - deletedFileId = storedFileRepository.save(createFile(alice, "/docs", "deleted.zip", false, "application/zip", 32L, LocalDateTime.now())).getId(); + foreignFileId = storedFileRepository.save(createBlobBackedFile( + bob, + "/docs", + "foreign.zip", + "application/zip", + "foreign-zip", + createZipArchive(Map.of("foreign.txt", "blocked")) + )).getId(); + deletedFileId = storedFileRepository.save(createBlobBackedFile( + alice, + "/docs", + "deleted.zip", + "application/zip", + "deleted-zip", + createZipArchive(Map.of("deleted.txt", "gone")), + LocalDateTime.now() + )).getId(); } @Test @@ -132,6 +213,9 @@ class BackgroundTaskV2ControllerIntegrationTest { .andExpect(jsonPath("$.data.publicStateJson", containsString("\"fileId\":" + archiveDirectoryId))) .andExpect(jsonPath("$.data.publicStateJson", containsString("\"path\":\"/docs/archive\""))) .andExpect(jsonPath("$.data.publicStateJson", containsString("\"directory\":true"))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\""))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0"))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":4"))) .andReturn() .getResponse() .getContentAsString(); @@ -148,6 +232,11 @@ class BackgroundTaskV2ControllerIntegrationTest { .andExpect(status().isOk()) .andExpect(jsonPath("$.data.type").value("EXTRACT")) .andExpect(jsonPath("$.data.status").value("QUEUED")) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputPath\":\"/docs\""))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputDirectoryName\":\"extract\""))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\""))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0"))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3"))) .andReturn() .getResponse() .getContentAsString(); @@ -164,6 +253,9 @@ class BackgroundTaskV2ControllerIntegrationTest { .andExpect(status().isOk()) .andExpect(jsonPath("$.data.type").value("MEDIA_META")) .andExpect(jsonPath("$.data.status").value("QUEUED")) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\""))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0"))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":2"))) .andReturn() .getResponse() .getContentAsString(); @@ -187,11 +279,13 @@ class BackgroundTaskV2ControllerIntegrationTest { mockMvc.perform(delete("/api/v2/tasks/{id}", extractId).with(user("alice"))) .andExpect(status().isOk()) - .andExpect(jsonPath("$.data.status").value("CANCELLED")); + .andExpect(jsonPath("$.data.status").value("CANCELLED")) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"cancelled\""))); BackgroundTask cancelled = backgroundTaskRepository.findById(extractId).orElseThrow(); assertThat(cancelled.getStatus()).isEqualTo(BackgroundTaskStatus.CANCELLED); assertThat(cancelled.getFinishedAt()).isNotNull(); + assertThat(cancelled.getPublicStateJson()).contains("\"phase\":\"cancelled\""); } @Test @@ -216,6 +310,289 @@ class BackgroundTaskV2ControllerIntegrationTest { mockMvc.perform(delete("/api/v2/tasks/{id}", taskId).with(user("bob"))) .andExpect(status().isNotFound()); + + mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("bob"))) + .andExpect(status().isNotFound()); + } + + @Test + void shouldRejectExtractTaskForNonZipCompatibleArchive() throws Exception { + mockMvc.perform(post("/api/v2/tasks/extract") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "path": "/docs/backup.7z" + } + """.formatted(unsupportedExtractFileId))) + .andExpect(status().isBadRequest()); + } + + @Test + void shouldCompleteArchiveTaskThroughWorkerAndExposeTerminalState() throws Exception { + String response = mockMvc.perform(post("/api/v2/tasks/archive") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "path": "/docs/archive-source.txt" + } + """.formatted(archiveFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue(); + + int processedCount = backgroundTaskWorker.processQueuedTasks(5); + + assertThat(processedCount).isEqualTo(1); + String taskResponse = mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice"))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.status").value("COMPLETED")) + .andReturn() + .getResponse() + .getContentAsString(); + + Map publicState = readPublicState(taskResponse); + assertThat(publicState).containsEntry("worker", "archive"); + assertThat(publicState).containsEntry("archivedFilename", "archive-source.txt.zip"); + assertThat(publicState).containsEntry("archivedPath", "/docs"); + assertThat(publicState).containsEntry("phase", "completed"); + assertThat(publicState).containsEntry("attemptCount", 1); + assertThat(publicState).containsEntry("maxAttempts", 4); + assertThat(publicState).containsEntry("processedFileCount", 1); + assertThat(publicState).containsEntry("totalFileCount", 1); + assertThat(publicState).containsEntry("processedDirectoryCount", 0); + assertThat(publicState).containsEntry("totalDirectoryCount", 0); + assertThat(publicState).containsEntry("progressPercent", 100); + assertThat(publicState.get("heartbeatAt")).isNotNull(); + assertThat(publicState).doesNotContainKey("workerOwner"); + assertThat(publicState).doesNotContainKey("leaseExpiresAt"); + assertThat(publicState.get("archivedFileId")).isNotNull(); + assertThat(publicState.get("archiveSize")).isNotNull(); + + BackgroundTask completed = backgroundTaskRepository.findById(taskId).orElseThrow(); + assertThat(completed.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED); + assertThat(completed.getFinishedAt()).isNotNull(); + assertThat(completed.getErrorMessage()).isNull(); + assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs", "archive-source.txt.zip")).isPresent(); + } + + @Test + void shouldCompleteExtractTaskThroughWorkerAndExposeTerminalState() throws Exception { + String response = mockMvc.perform(post("/api/v2/tasks/extract") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "path": "/docs/extract.zip" + } + """.formatted(extractFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue(); + + int processedCount = backgroundTaskWorker.processQueuedTasks(5); + + assertThat(processedCount).isEqualTo(1); + String taskResponse = mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice"))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.status").value("COMPLETED")) + .andReturn() + .getResponse() + .getContentAsString(); + + Map publicState = readPublicState(taskResponse); + assertThat(publicState).containsEntry("worker", "extract"); + assertThat(publicState).containsEntry("extractedPath", "/docs/extract"); + assertThat(publicState).containsEntry("extractedFileCount", 2); + assertThat(publicState).containsEntry("extractedDirectoryCount", 2); + assertThat(publicState).containsEntry("phase", "completed"); + assertThat(publicState).containsEntry("attemptCount", 1); + assertThat(publicState).containsEntry("maxAttempts", 3); + assertThat(publicState).containsEntry("processedFileCount", 2); + assertThat(publicState).containsEntry("totalFileCount", 2); + assertThat(publicState).containsEntry("processedDirectoryCount", 2); + assertThat(publicState).containsEntry("totalDirectoryCount", 2); + assertThat(publicState).containsEntry("progressPercent", 100); + assertThat(publicState.get("heartbeatAt")).isNotNull(); + assertThat(publicState).doesNotContainKey("workerOwner"); + assertThat(publicState).doesNotContainKey("leaseExpiresAt"); + + BackgroundTask completed = backgroundTaskRepository.findById(taskId).orElseThrow(); + assertThat(completed.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED); + assertThat(completed.getFinishedAt()).isNotNull(); + assertThat(completed.getErrorMessage()).isNull(); + assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs/extract", "notes.txt")).isPresent(); + assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs/extract/nested", "todo.txt")).isPresent(); + } + + @Test + void shouldMarkExtractTaskFailedWhenWorkerHitsInvalidArchiveContent() throws Exception { + String response = mockMvc.perform(post("/api/v2/tasks/extract") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "path": "/docs/broken.zip" + } + """.formatted(invalidExtractFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue(); + + int processedCount = backgroundTaskWorker.processQueuedTasks(5); + + assertThat(processedCount).isEqualTo(1); + mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice"))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.status").value("FAILED")) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"failed\""))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":1"))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3"))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"failureCategory\":\"DATA_STATE\""))) + .andExpect(jsonPath("$.data.errorMessage").value("extract task only supports zip-compatible archives")); + + BackgroundTask failed = backgroundTaskRepository.findById(taskId).orElseThrow(); + assertThat(failed.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED); + assertThat(failed.getFinishedAt()).isNotNull(); + assertThat(failed.getErrorMessage()).isEqualTo("extract task only supports zip-compatible archives"); + } + + @Test + void shouldRetryFailedTaskAndResetStateToQueued() throws Exception { + String response = mockMvc.perform(post("/api/v2/tasks/extract") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "path": "/docs/broken.zip" + } + """.formatted(invalidExtractFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue(); + backgroundTaskWorker.processQueuedTasks(5); + + String retryResponse = mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("alice"))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.status").value("QUEUED")) + .andExpect(jsonPath("$.data.errorMessage").doesNotExist()) + .andExpect(jsonPath("$.data.finishedAt").doesNotExist()) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\""))) + .andReturn() + .getResponse() + .getContentAsString(); + + Map publicState = readPublicState(retryResponse); + assertThat(publicState).containsEntry("phase", "queued"); + assertThat(publicState).containsEntry("outputPath", "/docs"); + assertThat(publicState).containsEntry("outputDirectoryName", "broken"); + assertThat(publicState).containsEntry("attemptCount", 0); + assertThat(publicState).containsEntry("maxAttempts", 3); + assertThat(publicState).doesNotContainKey("worker"); + assertThat(publicState).doesNotContainKey("processedFileCount"); + assertThat(publicState).doesNotContainKey("totalFileCount"); + + BackgroundTask retried = backgroundTaskRepository.findById(taskId).orElseThrow(); + assertThat(retried.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED); + assertThat(retried.getFinishedAt()).isNull(); + assertThat(retried.getErrorMessage()).isNull(); + } + + @Test + void shouldRejectRetryForNonFailedTask() throws Exception { + String response = mockMvc.perform(post("/api/v2/tasks/archive") + .with(user("alice")) + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "fileId": %d, + "path": "/docs/archive-source.txt" + } + """.formatted(archiveFileId))) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue(); + + mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("alice"))) + .andExpect(status().isBadRequest()); + } + + @Test + void shouldRecoverOnlyExpiredRunningTaskBackToQueuedOnStartup() throws Exception { + BackgroundTask expired = new BackgroundTask(); + expired.setType(BackgroundTaskType.EXTRACT); + expired.setStatus(BackgroundTaskStatus.RUNNING); + expired.setUserId(aliceId); + expired.setCorrelationId("recover-1"); + expired.setAttemptCount(1); + expired.setMaxAttempts(3); + expired.setLeaseOwner("worker-stale"); + expired.setLeaseExpiresAt(LocalDateTime.now().minusMinutes(2)); + expired.setHeartbeatAt(LocalDateTime.now().minusMinutes(3)); + expired.setPublicStateJson(""" + {"fileId":%d,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-stale","attemptCount":1,"maxAttempts":3} + """.formatted(extractFileId)); + expired.setPrivateStateJson(""" + {"fileId":%d,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"} + """.formatted(extractFileId)); + expired.setErrorMessage("stale worker"); + expired.setFinishedAt(LocalDateTime.now()); + expired = backgroundTaskRepository.save(expired); + + BackgroundTask fresh = new BackgroundTask(); + fresh.setType(BackgroundTaskType.EXTRACT); + fresh.setStatus(BackgroundTaskStatus.RUNNING); + fresh.setUserId(aliceId); + fresh.setCorrelationId("recover-2"); + fresh.setAttemptCount(1); + fresh.setMaxAttempts(3); + fresh.setLeaseOwner("worker-live"); + fresh.setLeaseExpiresAt(LocalDateTime.now().plusMinutes(5)); + fresh.setHeartbeatAt(LocalDateTime.now()); + fresh.setPublicStateJson(""" + {"fileId":%d,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-live","attemptCount":1,"maxAttempts":3} + """.formatted(extractFileId)); + fresh.setPrivateStateJson(""" + {"fileId":%d,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"} + """.formatted(extractFileId)); + fresh = backgroundTaskRepository.save(fresh); + + backgroundTaskStartupRecovery.recoverOnStartup(); + + mockMvc.perform(get("/api/v2/tasks/{id}", expired.getId()).with(user("alice"))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.status").value("QUEUED")) + .andExpect(jsonPath("$.data.errorMessage").doesNotExist()) + .andExpect(jsonPath("$.data.finishedAt").doesNotExist()) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\""))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":1"))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3"))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputPath\":\"/docs\""))) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputDirectoryName\":\"extract\""))) + .andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"worker\"")))) + .andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"workerOwner\"")))) + .andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"leaseExpiresAt\"")))); + + mockMvc.perform(get("/api/v2/tasks/{id}", fresh.getId()).with(user("alice"))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.status").value("RUNNING")) + .andExpect(jsonPath("$.data.publicStateJson", containsString("\"workerOwner\":\"worker-live\""))); } @Test @@ -274,4 +651,54 @@ class BackgroundTaskV2ControllerIntegrationTest { file.setDeletedAt(deletedAt); return file; } + + private StoredFile createBlobBackedFile(User user, + String path, + String filename, + String contentType, + String objectKeySuffix, + byte[] content) { + return createBlobBackedFile(user, path, filename, contentType, objectKeySuffix, content, null); + } + + private StoredFile createBlobBackedFile(User user, + String path, + String filename, + String contentType, + String objectKeySuffix, + byte[] content, + LocalDateTime deletedAt) { + String objectKey = "blobs/test-background-task/" + objectKeySuffix; + fileContentStorage.storeBlob(objectKey, contentType, content); + + FileBlob blob = new FileBlob(); + blob.setObjectKey(objectKey); + blob.setContentType(contentType); + blob.setSize((long) content.length); + blob = fileBlobRepository.save(blob); + + StoredFile file = createFile(user, path, filename, false, contentType, (long) content.length, deletedAt); + file.setBlob(blob); + return file; + } + + private byte[] createZipArchive(Map entries) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) { + for (Map.Entry entry : entries.entrySet()) { + zipOutputStream.putNextEntry(new ZipEntry(entry.getKey())); + if (!entry.getKey().endsWith("/")) { + zipOutputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); + } + zipOutputStream.closeEntry(); + } + } + return outputStream.toByteArray(); + } + + private Map readPublicState(String taskResponse) throws Exception { + String publicStateJson = JsonPath.read(taskResponse, "$.data.publicStateJson"); + return objectMapper.readValue(publicStateJson, new TypeReference>() { + }); + } } diff --git a/backend/src/test/java/com/yoyuzh/auth/AuthServiceTest.java b/backend/src/test/java/com/yoyuzh/auth/AuthServiceTest.java index 4745304..bd5e0ab 100644 --- a/backend/src/test/java/com/yoyuzh/auth/AuthServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/auth/AuthServiceTest.java @@ -7,8 +7,8 @@ import com.yoyuzh.auth.dto.UpdateUserAvatarRequest; import com.yoyuzh.auth.dto.UpdateUserPasswordRequest; import com.yoyuzh.auth.dto.UpdateUserProfileRequest; import com.yoyuzh.common.BusinessException; -import com.yoyuzh.files.FileService; -import com.yoyuzh.files.InitiateUploadResponse; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.upload.InitiateUploadResponse; import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.PreparedUpload; import org.junit.jupiter.api.Test; diff --git a/backend/src/test/java/com/yoyuzh/auth/AuthSingleDeviceIntegrationTest.java b/backend/src/test/java/com/yoyuzh/auth/AuthSingleDeviceIntegrationTest.java index 6ae30a7..7e726ea 100644 --- a/backend/src/test/java/com/yoyuzh/auth/AuthSingleDeviceIntegrationTest.java +++ b/backend/src/test/java/com/yoyuzh/auth/AuthSingleDeviceIntegrationTest.java @@ -39,7 +39,7 @@ class AuthSingleDeviceIntegrationTest { private UserRepository userRepository; @Autowired - private com.yoyuzh.files.StoredFileRepository storedFileRepository; + private com.yoyuzh.files.core.StoredFileRepository storedFileRepository; @Autowired private RefreshTokenRepository refreshTokenRepository; diff --git a/backend/src/test/java/com/yoyuzh/auth/DevBootstrapDataInitializerTest.java b/backend/src/test/java/com/yoyuzh/auth/DevBootstrapDataInitializerTest.java index cc1ff46..fe7ee20 100644 --- a/backend/src/test/java/com/yoyuzh/auth/DevBootstrapDataInitializerTest.java +++ b/backend/src/test/java/com/yoyuzh/auth/DevBootstrapDataInitializerTest.java @@ -1,7 +1,7 @@ package com.yoyuzh.auth; -import com.yoyuzh.files.FileService; -import com.yoyuzh.files.StoredFileRepository; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFileRepository; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; diff --git a/backend/src/test/java/com/yoyuzh/files/BackgroundTaskServiceTest.java b/backend/src/test/java/com/yoyuzh/files/BackgroundTaskServiceTest.java deleted file mode 100644 index 8e7d97e..0000000 --- a/backend/src/test/java/com/yoyuzh/files/BackgroundTaskServiceTest.java +++ /dev/null @@ -1,246 +0,0 @@ -package com.yoyuzh.files; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yoyuzh.api.v2.ApiV2Exception; -import com.yoyuzh.auth.User; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class BackgroundTaskServiceTest { - - @Mock - private BackgroundTaskRepository backgroundTaskRepository; - - @Mock - private StoredFileRepository storedFileRepository; - - private BackgroundTaskService backgroundTaskService; - - @BeforeEach - void setUp() { - backgroundTaskService = new BackgroundTaskService(backgroundTaskRepository, storedFileRepository, new ObjectMapper()); - } - - @Test - void shouldRejectTaskCreationForForeignFile() { - User user = createUser(7L); - when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(99L, 7L)).thenReturn(Optional.empty()); - - assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( - user, - BackgroundTaskType.ARCHIVE, - 99L, - "/docs/foreign.txt", - null - )).isInstanceOf(ApiV2Exception.class) - .hasMessage("file not found"); - } - - @Test - void shouldRejectTaskCreationForDeletedFile() { - User user = createUser(7L); - when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(100L, 7L)).thenReturn(Optional.empty()); - - assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( - user, - BackgroundTaskType.ARCHIVE, - 100L, - "/docs/deleted.txt", - null - )).isInstanceOf(ApiV2Exception.class) - .hasMessage("file not found"); - } - - @Test - void shouldRejectTaskCreationWhenRequestedPathDoesNotMatchFile() { - User user = createUser(7L); - StoredFile file = createStoredFile(11L, user, "/docs", "real.txt", false, "text/plain", 3L); - when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(file)); - - assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( - user, - BackgroundTaskType.ARCHIVE, - 11L, - "/docs/fake.txt", - null - )).isInstanceOf(ApiV2Exception.class) - .hasMessage("task path does not match file path"); - } - - @Test - void shouldRejectExtractTaskForDirectory() { - User user = createUser(7L); - StoredFile directory = createStoredFile(12L, user, "/", "bundle", true, null, 0L); - when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(12L, 7L)).thenReturn(Optional.of(directory)); - - assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( - user, - BackgroundTaskType.EXTRACT, - 12L, - "/bundle", - null - )).isInstanceOf(ApiV2Exception.class) - .hasMessage("task target type is not supported"); - } - - @Test - void shouldRejectMediaMetadataTaskForNonMediaFile() { - User user = createUser(7L); - StoredFile file = createStoredFile(13L, user, "/docs", "notes.txt", false, "text/plain", 9L); - when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(13L, 7L)).thenReturn(Optional.of(file)); - - assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( - user, - BackgroundTaskType.MEDIA_META, - 13L, - "/docs/notes.txt", - null - )).isInstanceOf(ApiV2Exception.class) - .hasMessage("media metadata task only supports media files"); - } - - @Test - void shouldCreateTaskStateFromServerFilePath() { - User user = createUser(7L); - StoredFile file = createStoredFile(14L, user, "/docs", "photo.png", false, "image/png", 15L); - when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(14L, 7L)).thenReturn(Optional.of(file)); - when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); - - BackgroundTask task = backgroundTaskService.createQueuedFileTask( - user, - BackgroundTaskType.MEDIA_META, - 14L, - "/docs/photo.png", - "media-1" - ); - - assertThat(task.getPublicStateJson()).contains("\"fileId\":14"); - assertThat(task.getPublicStateJson()).contains("\"path\":\"/docs/photo.png\""); - assertThat(task.getPublicStateJson()).contains("\"filename\":\"photo.png\""); - assertThat(task.getPublicStateJson()).contains("\"directory\":false"); - assertThat(task.getPublicStateJson()).contains("\"contentType\":\"image/png\""); - assertThat(task.getPublicStateJson()).contains("\"size\":15"); - assertThat(task.getPrivateStateJson()).contains("\"taskType\":\"MEDIA_META\""); - } - - @Test - void shouldClaimQueuedTaskOnlyWhenRepositoryTransitionSucceeds() { - BackgroundTask task = createTask(1L, BackgroundTaskStatus.RUNNING); - when(backgroundTaskRepository.claimQueuedTask( - eq(1L), - eq(BackgroundTaskStatus.QUEUED), - eq(BackgroundTaskStatus.RUNNING), - any() - )).thenReturn(1); - when(backgroundTaskRepository.findById(1L)).thenReturn(Optional.of(task)); - - Optional result = backgroundTaskService.claimQueuedTask(1L); - - assertThat(result).containsSame(task); - } - - @Test - void shouldNotClaimTaskWhenRepositoryTransitionWasSkipped() { - when(backgroundTaskRepository.claimQueuedTask( - eq(2L), - eq(BackgroundTaskStatus.QUEUED), - eq(BackgroundTaskStatus.RUNNING), - any() - )).thenReturn(0); - - Optional result = backgroundTaskService.claimQueuedTask(2L); - - assertThat(result).isEmpty(); - } - - @Test - void shouldCompleteRunningWorkerTaskAndMergePublicState() { - BackgroundTask task = createTask(3L, BackgroundTaskStatus.RUNNING); - task.setPublicStateJson("{\"fileId\":11}"); - when(backgroundTaskRepository.findById(3L)).thenReturn(Optional.of(task)); - when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); - - BackgroundTask result = backgroundTaskService.markWorkerTaskCompleted(3L, Map.of("worker", "noop")); - - assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED); - assertThat(result.getFinishedAt()).isNotNull(); - assertThat(result.getErrorMessage()).isNull(); - assertThat(result.getPublicStateJson()).contains("\"fileId\":11"); - assertThat(result.getPublicStateJson()).contains("\"worker\":\"noop\""); - } - - @Test - void shouldRecordWorkerFailureMessage() { - BackgroundTask task = createTask(4L, BackgroundTaskStatus.RUNNING); - when(backgroundTaskRepository.findById(4L)).thenReturn(Optional.of(task)); - when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); - - BackgroundTask result = backgroundTaskService.markWorkerTaskFailed(4L, "media parser unavailable"); - - assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED); - assertThat(result.getFinishedAt()).isNotNull(); - assertThat(result.getErrorMessage()).isEqualTo("media parser unavailable"); - } - - @Test - void shouldFindQueuedTaskIdsInCreatedOrderLimit() { - BackgroundTask first = createTask(5L, BackgroundTaskStatus.QUEUED); - BackgroundTask second = createTask(6L, BackgroundTaskStatus.QUEUED); - when(backgroundTaskRepository.findByStatusOrderByCreatedAtAsc(eq(BackgroundTaskStatus.QUEUED), any())) - .thenReturn(List.of(first, second)); - - List result = backgroundTaskService.findQueuedTaskIds(2); - - assertThat(result).containsExactly(5L, 6L); - } - - private BackgroundTask createTask(Long id, BackgroundTaskStatus status) { - BackgroundTask task = new BackgroundTask(); - task.setId(id); - task.setType(BackgroundTaskType.MEDIA_META); - task.setStatus(status); - task.setUserId(7L); - task.setPublicStateJson("{}"); - task.setPrivateStateJson("{}"); - return task; - } - - private User createUser(Long id) { - User user = new User(); - user.setId(id); - user.setUsername("alice"); - return user; - } - - private StoredFile createStoredFile(Long id, - User user, - String path, - String filename, - boolean directory, - String contentType, - Long size) { - StoredFile file = new StoredFile(); - file.setId(id); - file.setUser(user); - file.setPath(path); - file.setFilename(filename); - file.setDirectory(directory); - file.setContentType(contentType); - file.setSize(size); - return file; - } -} diff --git a/backend/src/test/java/com/yoyuzh/files/BackgroundTaskWorkerTest.java b/backend/src/test/java/com/yoyuzh/files/BackgroundTaskWorkerTest.java deleted file mode 100644 index dccca1b..0000000 --- a/backend/src/test/java/com/yoyuzh/files/BackgroundTaskWorkerTest.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.yoyuzh.files; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class BackgroundTaskWorkerTest { - - @Mock - private BackgroundTaskService backgroundTaskService; - @Mock - private BackgroundTaskHandler backgroundTaskHandler; - - private BackgroundTaskWorker backgroundTaskWorker; - - @BeforeEach - void setUp() { - backgroundTaskWorker = new BackgroundTaskWorker(backgroundTaskService, List.of(backgroundTaskHandler)); - } - - @Test - void shouldClaimAndCompleteQueuedTaskThroughNoopHandler() { - BackgroundTask task = createTask(1L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING); - when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L)); - when(backgroundTaskService.claimQueuedTask(1L)).thenReturn(Optional.of(task)); - when(backgroundTaskHandler.supports(BackgroundTaskType.ARCHIVE)).thenReturn(true); - when(backgroundTaskHandler.handle(task)).thenReturn(new BackgroundTaskHandlerResult(Map.of("worker", "noop"))); - - int processedCount = backgroundTaskWorker.processQueuedTasks(5); - - assertThat(processedCount).isEqualTo(1); - verify(backgroundTaskHandler).handle(task); - verify(backgroundTaskService).markWorkerTaskCompleted(1L, Map.of("worker", "noop")); - } - - @Test - void shouldSkipTaskThatWasNotClaimed() { - when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L)); - when(backgroundTaskService.claimQueuedTask(1L)).thenReturn(Optional.empty()); - - int processedCount = backgroundTaskWorker.processQueuedTasks(5); - - assertThat(processedCount).isZero(); - verify(backgroundTaskHandler, never()).handle(org.mockito.ArgumentMatchers.any()); - } - - @Test - void shouldMarkTaskFailedWhenHandlerThrows() { - BackgroundTask task = createTask(2L, BackgroundTaskType.MEDIA_META, BackgroundTaskStatus.RUNNING); - when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(2L)); - when(backgroundTaskService.claimQueuedTask(2L)).thenReturn(Optional.of(task)); - when(backgroundTaskHandler.supports(BackgroundTaskType.MEDIA_META)).thenReturn(true); - when(backgroundTaskHandler.handle(task)).thenThrow(new IllegalStateException("media parser unavailable")); - - int processedCount = backgroundTaskWorker.processQueuedTasks(5); - - assertThat(processedCount).isEqualTo(1); - verify(backgroundTaskService).markWorkerTaskFailed(2L, "media parser unavailable"); - } - - private BackgroundTask createTask(Long id, BackgroundTaskType type, BackgroundTaskStatus status) { - BackgroundTask task = new BackgroundTask(); - task.setId(id); - task.setType(type); - task.setStatus(status); - task.setUserId(7L); - task.setPublicStateJson("{}"); - task.setPrivateStateJson("{}"); - return task; - } -} diff --git a/backend/src/test/java/com/yoyuzh/files/FileBlobBackfillServiceTest.java b/backend/src/test/java/com/yoyuzh/files/core/FileBlobBackfillServiceTest.java similarity index 99% rename from backend/src/test/java/com/yoyuzh/files/FileBlobBackfillServiceTest.java rename to backend/src/test/java/com/yoyuzh/files/core/FileBlobBackfillServiceTest.java index 3e7bae7..87108dc 100644 --- a/backend/src/test/java/com/yoyuzh/files/FileBlobBackfillServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/files/core/FileBlobBackfillServiceTest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.auth.User; import com.yoyuzh.files.storage.FileContentStorage; diff --git a/backend/src/test/java/com/yoyuzh/files/FileEntityBackfillServiceTest.java b/backend/src/test/java/com/yoyuzh/files/core/FileEntityBackfillServiceTest.java similarity index 95% rename from backend/src/test/java/com/yoyuzh/files/FileEntityBackfillServiceTest.java rename to backend/src/test/java/com/yoyuzh/files/core/FileEntityBackfillServiceTest.java index eded0fe..6dcaecd 100644 --- a/backend/src/test/java/com/yoyuzh/files/FileEntityBackfillServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/files/core/FileEntityBackfillServiceTest.java @@ -1,6 +1,10 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.auth.User; +import com.yoyuzh.files.policy.StoragePolicy; +import com.yoyuzh.files.policy.StoragePolicyCredentialMode; +import com.yoyuzh.files.policy.StoragePolicyService; +import com.yoyuzh.files.policy.StoragePolicyType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/backend/src/test/java/com/yoyuzh/files/FileServiceEdgeCaseTest.java b/backend/src/test/java/com/yoyuzh/files/core/FileServiceEdgeCaseTest.java similarity index 98% rename from backend/src/test/java/com/yoyuzh/files/FileServiceEdgeCaseTest.java rename to backend/src/test/java/com/yoyuzh/files/core/FileServiceEdgeCaseTest.java index 43ad512..3402fa5 100644 --- a/backend/src/test/java/com/yoyuzh/files/FileServiceEdgeCaseTest.java +++ b/backend/src/test/java/com/yoyuzh/files/core/FileServiceEdgeCaseTest.java @@ -1,9 +1,11 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.admin.AdminMetricsService; import com.yoyuzh.auth.User; import com.yoyuzh.common.BusinessException; import com.yoyuzh.config.FileStorageProperties; +import com.yoyuzh.files.share.FileShareLinkRepository; +import com.yoyuzh.files.upload.InitiateUploadRequest; import com.yoyuzh.files.storage.FileContentStorage; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/backend/src/test/java/com/yoyuzh/files/FileServiceTest.java b/backend/src/test/java/com/yoyuzh/files/core/FileServiceTest.java similarity index 82% rename from backend/src/test/java/com/yoyuzh/files/FileServiceTest.java rename to backend/src/test/java/com/yoyuzh/files/core/FileServiceTest.java index 69a585c..ea8fdf3 100644 --- a/backend/src/test/java/com/yoyuzh/files/FileServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/files/core/FileServiceTest.java @@ -1,9 +1,19 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.admin.AdminMetricsService; import com.yoyuzh.auth.User; import com.yoyuzh.common.BusinessException; import com.yoyuzh.config.FileStorageProperties; +import com.yoyuzh.files.policy.StoragePolicy; +import com.yoyuzh.files.policy.StoragePolicyCredentialMode; +import com.yoyuzh.files.policy.StoragePolicyService; +import com.yoyuzh.files.policy.StoragePolicyType; +import com.yoyuzh.files.share.CreateFileShareLinkResponse; +import com.yoyuzh.files.share.FileShareLink; +import com.yoyuzh.files.share.FileShareLinkRepository; +import com.yoyuzh.files.upload.CompleteUploadRequest; +import com.yoyuzh.files.upload.InitiateUploadRequest; +import com.yoyuzh.files.upload.InitiateUploadResponse; import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.PreparedUpload; import org.junit.jupiter.api.BeforeEach; @@ -19,6 +29,8 @@ import org.springframework.http.ResponseEntity; import org.springframework.mock.web.MockMultipartFile; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Clock; @@ -29,7 +41,10 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -722,6 +737,138 @@ class FileServiceTest { verify(fileContentStorage).readBlob("blobs/blob-13"); } + @Test + void shouldBuildZipBytesForDirectoryForBackgroundArchiveReuse() throws Exception { + User user = createUser(7L); + StoredFile directory = createDirectory(10L, user, "/docs", "archive"); + StoredFile childDirectory = createDirectory(11L, user, "/docs/archive", "nested"); + StoredFile childFile = createFile(12L, user, "/docs/archive", "notes.txt"); + StoredFile nestedFile = createFile(13L, user, "/docs/archive/nested", "todo.txt"); + + when(storedFileRepository.findByUserIdAndPathEqualsOrDescendant(7L, "/docs/archive")) + .thenReturn(List.of(childDirectory, childFile, nestedFile)); + when(fileContentStorage.readBlob("blobs/blob-12")) + .thenReturn("hello".getBytes(StandardCharsets.UTF_8)); + when(fileContentStorage.readBlob("blobs/blob-13")) + .thenReturn("world".getBytes(StandardCharsets.UTF_8)); + + byte[] archiveBytes = fileService.buildArchiveBytes(directory); + + Map entries = readZipEntries(archiveBytes); + + assertThat(entries).containsEntry("archive/", ""); + assertThat(entries).containsEntry("archive/nested/", ""); + assertThat(entries).containsEntry("archive/notes.txt", "hello"); + assertThat(entries).containsEntry("archive/nested/todo.txt", "world"); + verify(fileContentStorage).readBlob("blobs/blob-12"); + verify(fileContentStorage).readBlob("blobs/blob-13"); + } + + @Test + void shouldBuildZipBytesForSingleFileForBackgroundArchiveReuse() throws Exception { + User user = createUser(7L); + StoredFile file = createFile(12L, user, "/docs", "notes.txt"); + when(fileContentStorage.readBlob("blobs/blob-12")) + .thenReturn("hello".getBytes(StandardCharsets.UTF_8)); + + byte[] archiveBytes = fileService.buildArchiveBytes(file); + + Map entries = readZipEntries(archiveBytes); + + assertThat(entries).containsEntry("notes.txt", "hello"); + verify(fileContentStorage).readBlob("blobs/blob-12"); + verify(storedFileRepository, never()).findByUserIdAndPathEqualsOrDescendant(any(), any()); + } + + @Test + void shouldReadZipCompatibleArchiveForExtractTaskReuse() throws Exception { + User user = createUser(7L); + StoredFile archive = createFile(20L, user, "/docs", "extract.zip", createBlob(20L, "blobs/blob-20", 64L, "application/zip")); + when(fileContentStorage.readBlob("blobs/blob-20")).thenReturn(createZipArchive(Map.of( + "archive/", "", + "archive/nested/", "", + "archive/notes.txt", "hello", + "archive/nested/todo.txt", "world" + ))); + + FileService.ZipCompatibleArchive zipArchive = fileService.readZipCompatibleArchive(archive); + + assertThat(zipArchive.commonRootDirectoryName()).isEqualTo("archive"); + assertThat(zipArchive.entries()) + .extracting(FileService.ZipCompatibleArchiveEntry::relativePath, FileService.ZipCompatibleArchiveEntry::directory) + .containsExactlyInAnyOrder( + org.assertj.core.groups.Tuple.tuple("archive", true), + org.assertj.core.groups.Tuple.tuple("archive/nested", true), + org.assertj.core.groups.Tuple.tuple("archive/notes.txt", false), + org.assertj.core.groups.Tuple.tuple("archive/nested/todo.txt", false) + ); + Map fileEntries = zipArchive.entries().stream() + .filter(entry -> !entry.directory()) + .collect(java.util.stream.Collectors.toMap( + FileService.ZipCompatibleArchiveEntry::relativePath, + entry -> new String(entry.content(), StandardCharsets.UTF_8), + (left, right) -> left, + LinkedHashMap::new + )); + assertThat(fileEntries) + .containsEntry("archive/notes.txt", "hello") + .containsEntry("archive/nested/todo.txt", "world"); + verify(fileContentStorage).readBlob("blobs/blob-20"); + } + + @Test + void shouldRejectZipCompatibleArchiveWithTraversalEntry() throws Exception { + User user = createUser(7L); + StoredFile archive = createFile(21L, user, "/docs", "extract.zip", createBlob(21L, "blobs/blob-21", 32L, "application/zip")); + when(fileContentStorage.readBlob("blobs/blob-21")).thenReturn(createZipArchive(Map.of( + "../evil.txt", "oops" + ))); + + assertThatThrownBy(() -> fileService.readZipCompatibleArchive(archive)) + .isInstanceOf(BusinessException.class) + .hasMessage("压缩包内容不合法"); + } + + @Test + void shouldDeleteWrittenBlobsWhenBatchExternalImportFails() { + User user = createUser(8L); + StoredFile docs = createDirectory(300L, user, "/", "docs"); + when(storedFileRepository.findByUserIdAndPathAndFilename(8L, "/", "docs")).thenReturn(Optional.of(docs)); + when(storedFileRepository.existsByUserIdAndPathAndFilename(8L, "/docs", "first.txt")).thenReturn(false); + when(storedFileRepository.existsByUserIdAndPathAndFilename(8L, "/docs", "second.txt")).thenReturn(false); + when(fileBlobRepository.save(any(FileBlob.class))).thenAnswer(invocation -> { + FileBlob blob = invocation.getArgument(0); + blob.setId(System.nanoTime()); + return blob; + }); + when(storedFileRepository.save(any(StoredFile.class))) + .thenAnswer(invocation -> { + StoredFile file = invocation.getArgument(0); + if ("second.txt".equals(file.getFilename())) { + throw new BusinessException(com.yoyuzh.common.ErrorCode.UNKNOWN, "metadata save failed"); + } + file.setId(400L); + return file; + }); + + assertThatThrownBy(() -> fileService.importExternalFilesAtomically( + user, + List.of(), + List.of( + new FileService.ExternalFileImport("/docs", "first.txt", "text/plain", "first".getBytes(StandardCharsets.UTF_8)), + new FileService.ExternalFileImport("/docs", "second.txt", "text/plain", "second".getBytes(StandardCharsets.UTF_8)) + ) + )).isInstanceOf(BusinessException.class) + .hasMessage("metadata save failed"); + + var objectKeyCaptor = forClass(String.class); + verify(fileContentStorage, times(2)).storeBlob(objectKeyCaptor.capture(), eq("text/plain"), any(byte[].class)); + List writtenKeys = objectKeyCaptor.getAllValues(); + assertThat(writtenKeys).hasSize(2); + verify(fileContentStorage).deleteBlob(writtenKeys.get(0)); + verify(fileContentStorage).deleteBlob(writtenKeys.get(1)); + } + @Test void shouldCreateShareLinkForOwnedFile() { User user = createUser(7L); @@ -845,4 +992,37 @@ class FileServiceTest { policy.setDefaultPolicy(true); return policy; } + + private Map readZipEntries(byte[] archiveBytes) throws Exception { + Map entries = new LinkedHashMap<>(); + try (ZipInputStream zipInputStream = new ZipInputStream( + new ByteArrayInputStream(archiveBytes), StandardCharsets.UTF_8)) { + var entry = zipInputStream.getNextEntry(); + while (entry != null) { + entries.put(entry.getName(), entry.isDirectory() ? "" : new String(zipInputStream.readAllBytes(), StandardCharsets.UTF_8)); + entry = zipInputStream.getNextEntry(); + } + } + return entries; + } + + private byte[] createZipArchive(Map entries) throws IOException { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) { + Set createdEntries = new java.util.LinkedHashSet<>(); + for (Map.Entry entry : entries.entrySet()) { + if (!createdEntries.add(entry.getKey())) { + continue; + } + zipOutputStream.putNextEntry(new ZipEntry(entry.getKey())); + if (!entry.getKey().endsWith("/")) { + zipOutputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); + } + zipOutputStream.closeEntry(); + } + zipOutputStream.finish(); + return outputStream.toByteArray(); + } + } + } diff --git a/backend/src/test/java/com/yoyuzh/files/FileShareControllerIntegrationTest.java b/backend/src/test/java/com/yoyuzh/files/core/FileShareControllerIntegrationTest.java similarity index 99% rename from backend/src/test/java/com/yoyuzh/files/FileShareControllerIntegrationTest.java rename to backend/src/test/java/com/yoyuzh/files/core/FileShareControllerIntegrationTest.java index bb72180..9a43d37 100644 --- a/backend/src/test/java/com/yoyuzh/files/FileShareControllerIntegrationTest.java +++ b/backend/src/test/java/com/yoyuzh/files/core/FileShareControllerIntegrationTest.java @@ -1,8 +1,9 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.auth.User; import com.yoyuzh.auth.UserRepository; +import com.yoyuzh.files.share.FileShareLinkRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; diff --git a/backend/src/test/java/com/yoyuzh/files/RecycleBinControllerIntegrationTest.java b/backend/src/test/java/com/yoyuzh/files/core/RecycleBinControllerIntegrationTest.java similarity index 99% rename from backend/src/test/java/com/yoyuzh/files/RecycleBinControllerIntegrationTest.java rename to backend/src/test/java/com/yoyuzh/files/core/RecycleBinControllerIntegrationTest.java index 8656473..38f483c 100644 --- a/backend/src/test/java/com/yoyuzh/files/RecycleBinControllerIntegrationTest.java +++ b/backend/src/test/java/com/yoyuzh/files/core/RecycleBinControllerIntegrationTest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.core; import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.auth.User; diff --git a/backend/src/test/java/com/yoyuzh/files/FileEventPersistenceIntegrationTest.java b/backend/src/test/java/com/yoyuzh/files/events/FileEventPersistenceIntegrationTest.java similarity index 88% rename from backend/src/test/java/com/yoyuzh/files/FileEventPersistenceIntegrationTest.java rename to backend/src/test/java/com/yoyuzh/files/events/FileEventPersistenceIntegrationTest.java index 64fe015..0b32d2e 100644 --- a/backend/src/test/java/com/yoyuzh/files/FileEventPersistenceIntegrationTest.java +++ b/backend/src/test/java/com/yoyuzh/files/events/FileEventPersistenceIntegrationTest.java @@ -1,10 +1,19 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.events; import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.admin.AdminMetricsService; import com.yoyuzh.auth.User; import com.yoyuzh.common.BusinessException; import com.yoyuzh.config.FileStorageProperties; +import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.files.core.FileEntityRepository; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileEntityRepository; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.policy.StoragePolicyService; +import com.yoyuzh.files.share.FileShareLinkRepository; import com.yoyuzh.files.storage.FileContentStorage; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/backend/src/test/java/com/yoyuzh/files/StoragePolicyServiceTest.java b/backend/src/test/java/com/yoyuzh/files/policy/StoragePolicyServiceTest.java similarity index 97% rename from backend/src/test/java/com/yoyuzh/files/StoragePolicyServiceTest.java rename to backend/src/test/java/com/yoyuzh/files/policy/StoragePolicyServiceTest.java index dfb17eb..970082a 100644 --- a/backend/src/test/java/com/yoyuzh/files/StoragePolicyServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/files/policy/StoragePolicyServiceTest.java @@ -1,4 +1,4 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.policy; import com.yoyuzh.config.FileStorageProperties; import org.junit.jupiter.api.BeforeEach; @@ -56,7 +56,7 @@ class StoragePolicyServiceTest { StoragePolicyCapabilities capabilities = storagePolicyService.readCapabilities(policy); assertThat(capabilities.directUpload()).isTrue(); - assertThat(capabilities.multipartUpload()).isFalse(); + assertThat(capabilities.multipartUpload()).isTrue(); assertThat(capabilities.signedDownloadUrl()).isTrue(); assertThat(capabilities.serverProxyDownload()).isTrue(); assertThat(capabilities.requiresCors()).isTrue(); diff --git a/backend/src/test/java/com/yoyuzh/files/FileSearchServiceTest.java b/backend/src/test/java/com/yoyuzh/files/search/FileSearchServiceTest.java similarity index 97% rename from backend/src/test/java/com/yoyuzh/files/FileSearchServiceTest.java rename to backend/src/test/java/com/yoyuzh/files/search/FileSearchServiceTest.java index 7831776..7f48d0c 100644 --- a/backend/src/test/java/com/yoyuzh/files/FileSearchServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/files/search/FileSearchServiceTest.java @@ -1,7 +1,9 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.search; import com.yoyuzh.api.v2.ApiV2Exception; import com.yoyuzh.auth.User; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/backend/src/test/java/com/yoyuzh/files/storage/S3FileContentStorageTest.java b/backend/src/test/java/com/yoyuzh/files/storage/S3FileContentStorageTest.java index d828a7d..244a844 100644 --- a/backend/src/test/java/com/yoyuzh/files/storage/S3FileContentStorageTest.java +++ b/backend/src/test/java/com/yoyuzh/files/storage/S3FileContentStorageTest.java @@ -11,7 +11,13 @@ import org.mockito.junit.jupiter.MockitoExtension; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; @@ -24,9 +30,12 @@ import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest; import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest; import java.net.URL; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -160,6 +169,70 @@ class S3FileContentStorageTest { assertThat(deleteCaptor.getValue().key()).isEqualTo("users/7/docs/old.txt"); } + @Test + void createMultipartUploadStartsMultipartInConfiguredBucket() { + when(s3Client.createMultipartUpload(any(CreateMultipartUploadRequest.class))) + .thenReturn(CreateMultipartUploadResponse.builder().uploadId("upload-123").build()); + + String uploadId = storage.createMultipartUpload("blobs/object-1", "video/mp4"); + + assertThat(uploadId).isEqualTo("upload-123"); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CreateMultipartUploadRequest.class); + verify(s3Client).createMultipartUpload(requestCaptor.capture()); + assertThat(requestCaptor.getValue().bucket()).isEqualTo("demo-bucket"); + assertThat(requestCaptor.getValue().key()).isEqualTo("blobs/object-1"); + assertThat(requestCaptor.getValue().contentType()).isEqualTo("video/mp4"); + } + + @Test + void prepareMultipartPartUploadSignsUploadPartRequest() throws Exception { + PresignedUploadPartRequest presignedRequest = org.mockito.Mockito.mock(PresignedUploadPartRequest.class); + when(presignedRequest.url()).thenReturn(new URL("https://upload.example.com/blobs/object-1?partNumber=2")); + when(s3Presigner.presignUploadPart(any(UploadPartPresignRequest.class))).thenReturn(presignedRequest); + + PreparedUpload preparedUpload = storage.prepareMultipartPartUpload("blobs/object-1", "upload-123", 2, "video/mp4", 1024L); + + assertThat(preparedUpload.direct()).isTrue(); + assertThat(preparedUpload.method()).isEqualTo("PUT"); + assertThat(preparedUpload.uploadUrl()).contains("partNumber=2"); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(UploadPartPresignRequest.class); + verify(s3Presigner).presignUploadPart(requestCaptor.capture()); + assertThat(requestCaptor.getValue().uploadPartRequest().bucket()).isEqualTo("demo-bucket"); + assertThat(requestCaptor.getValue().uploadPartRequest().key()).isEqualTo("blobs/object-1"); + assertThat(requestCaptor.getValue().uploadPartRequest().partNumber()).isEqualTo(2); + assertThat(requestCaptor.getValue().uploadPartRequest().uploadId()).isEqualTo("upload-123"); + } + + @Test + void completeMultipartUploadSubmitsSortedCompletedParts() { + when(s3Client.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))) + .thenReturn(CompleteMultipartUploadResponse.builder().build()); + + storage.completeMultipartUpload("blobs/object-1", "upload-123", List.of( + new MultipartCompletedPart(2, "etag-2"), + new MultipartCompletedPart(1, "etag-1") + )); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CompleteMultipartUploadRequest.class); + verify(s3Client).completeMultipartUpload(requestCaptor.capture()); + assertThat(requestCaptor.getValue().bucket()).isEqualTo("demo-bucket"); + assertThat(requestCaptor.getValue().key()).isEqualTo("blobs/object-1"); + assertThat(requestCaptor.getValue().uploadId()).isEqualTo("upload-123"); + assertThat(requestCaptor.getValue().multipartUpload().parts()).extracting(CompletedPart::partNumber) + .containsExactly(1, 2); + } + + @Test + void abortMultipartUploadCancelsRemoteMultipartState() { + storage.abortMultipartUpload("blobs/object-1", "upload-123"); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(AbortMultipartUploadRequest.class); + verify(s3Client).abortMultipartUpload(requestCaptor.capture()); + assertThat(requestCaptor.getValue().bucket()).isEqualTo("demo-bucket"); + assertThat(requestCaptor.getValue().key()).isEqualTo("blobs/object-1"); + assertThat(requestCaptor.getValue().uploadId()).isEqualTo("upload-123"); + } + @Test void readFileFallsBackToLegacyObjectKeyWhenNeeded() { when(s3Client.headObject(HeadObjectRequest.builder().bucket("demo-bucket").key("users/7/docs/notes.txt").build())) diff --git a/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskArchiveHandlerTest.java b/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskArchiveHandlerTest.java new file mode 100644 index 0000000..54c37bd --- /dev/null +++ b/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskArchiveHandlerTest.java @@ -0,0 +1,234 @@ +package com.yoyuzh.files.tasks; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.auth.User; +import com.yoyuzh.auth.UserRepository; +import com.yoyuzh.files.core.FileBlob; +import com.yoyuzh.files.core.FileMetadataResponse; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.storage.FileContentStorage; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.zip.ZipInputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class BackgroundTaskArchiveHandlerTest { + + @Mock + private StoredFileRepository storedFileRepository; + @Mock + private UserRepository userRepository; + @Mock + private FileService fileService; + + private ArchiveBackgroundTaskHandler handler; + private ArgumentCaptor archiveBytesCaptor; + + @BeforeEach + void setUp() { + handler = new ArchiveBackgroundTaskHandler( + storedFileRepository, + userRepository, + fileService, + new ObjectMapper() + ); + archiveBytesCaptor = ArgumentCaptor.forClass(byte[].class); + } + + @Test + void shouldArchiveDirectoryAndImportZipIntoSameParentPath() throws Exception { + User user = createUser(7L); + StoredFile directory = createDirectory(11L, user, "/docs", "archive"); + StoredFile nestedDirectory = createDirectory(12L, user, "/docs/archive", "nested"); + StoredFile childFile = createFile(13L, user, "/docs/archive", "notes.txt", "text/plain", "blobs/blob-13"); + StoredFile nestedFile = createFile(14L, user, "/docs/archive/nested", "todo.txt", "text/plain", "blobs/blob-14"); + FileMetadataResponse importedArchive = new FileMetadataResponse( + 99L, + "archive.zip", + "/docs", + 123L, + "application/zip", + false, + null + ); + + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(directory)); + when(userRepository.findById(7L)).thenReturn(Optional.of(user)); + when(fileService.summarizeArchiveSource(directory)).thenReturn(new FileService.ArchiveSourceSummary(2, 2)); + when(fileService.buildArchiveBytes(eq(directory), any())).thenReturn(buildArchiveBytes(Map.of( + "archive/", "", + "archive/nested/", "", + "archive/notes.txt", "hello", + "archive/nested/todo.txt", "world" + ))); + when(fileService.importExternalFile(eq(user), eq("/docs"), eq("archive.zip"), eq("application/zip"), anyLong(), any(byte[].class))) + .thenReturn(importedArchive); + + BackgroundTaskHandlerResult result = handler.handle(createArchiveTask(11L, 7L)); + + verify(fileService).importExternalFile( + eq(user), + eq("/docs"), + eq("archive.zip"), + eq("application/zip"), + anyLong(), + archiveBytesCaptor.capture() + ); + + Map entries = readZipEntries(archiveBytesCaptor.getValue()); + + assertThat(entries).containsEntry("archive/", ""); + assertThat(entries).containsEntry("archive/nested/", ""); + assertThat(entries).containsEntry("archive/notes.txt", "hello"); + assertThat(entries).containsEntry("archive/nested/todo.txt", "world"); + assertThat(result.publicStatePatch()).containsEntry("worker", "archive"); + assertThat(result.publicStatePatch()).containsEntry("archivedFileId", 99L); + assertThat(result.publicStatePatch()).containsEntry("archivedFilename", "archive.zip"); + assertThat(result.publicStatePatch()).containsEntry("archivedPath", "/docs"); + assertThat(result.publicStatePatch()).containsEntry("processedFileCount", 2); + assertThat(result.publicStatePatch()).containsEntry("totalFileCount", 2); + assertThat(result.publicStatePatch()).containsEntry("processedDirectoryCount", 2); + assertThat(result.publicStatePatch()).containsEntry("totalDirectoryCount", 2); + verify(fileService).buildArchiveBytes(eq(directory), any()); + } + + @Test + void shouldArchiveSingleFileIntoZipWithoutLoadingDescendants() throws Exception { + User user = createUser(7L); + StoredFile file = createFile(21L, user, "/docs", "notes.txt", "text/plain", "blobs/blob-21"); + FileMetadataResponse importedArchive = new FileMetadataResponse( + 100L, + "notes.txt.zip", + "/docs", + 12L, + "application/zip", + false, + null + ); + + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(21L, 7L)).thenReturn(Optional.of(file)); + when(userRepository.findById(7L)).thenReturn(Optional.of(user)); + when(fileService.summarizeArchiveSource(file)).thenReturn(new FileService.ArchiveSourceSummary(1, 0)); + when(fileService.buildArchiveBytes(eq(file), any())).thenReturn(buildArchiveBytes(Map.of( + "notes.txt", "hello" + ))); + when(fileService.importExternalFile(eq(user), eq("/docs"), eq("notes.txt.zip"), eq("application/zip"), anyLong(), any(byte[].class))) + .thenReturn(importedArchive); + + handler.handle(createArchiveTask(21L, 7L)); + + verify(storedFileRepository, never()).findByUserIdAndPathEqualsOrDescendant(anyLong(), any()); + verify(fileService).importExternalFile( + eq(user), + eq("/docs"), + eq("notes.txt.zip"), + eq("application/zip"), + anyLong(), + archiveBytesCaptor.capture() + ); + + Map entries = readZipEntries(archiveBytesCaptor.getValue()); + assertThat(entries).containsEntry("notes.txt", "hello"); + } + + private BackgroundTask createArchiveTask(Long fileId, Long userId) { + BackgroundTask task = new BackgroundTask(); + task.setId(301L); + task.setType(BackgroundTaskType.ARCHIVE); + task.setStatus(BackgroundTaskStatus.RUNNING); + task.setUserId(userId); + task.setPublicStateJson("{\"fileId\":" + fileId + "}"); + task.setPrivateStateJson("{\"fileId\":" + fileId + ",\"taskType\":\"ARCHIVE\",\"outputPath\":\"/docs\",\"outputFilename\":\"" + + (fileId.equals(21L) ? "notes.txt.zip" : "archive.zip") + "\"}"); + return task; + } + + private User createUser(Long id) { + User user = new User(); + user.setId(id); + user.setUsername("alice"); + return user; + } + + private StoredFile createDirectory(Long id, User user, String path, String filename) { + StoredFile file = new StoredFile(); + file.setId(id); + file.setUser(user); + file.setPath(path); + file.setFilename(filename); + file.setDirectory(true); + file.setSize(0L); + return file; + } + + private StoredFile createFile(Long id, + User user, + String path, + String filename, + String contentType, + String objectKey) { + StoredFile file = new StoredFile(); + file.setId(id); + file.setUser(user); + file.setPath(path); + file.setFilename(filename); + file.setDirectory(false); + file.setContentType(contentType); + file.setSize(5L); + FileBlob blob = new FileBlob(); + blob.setId(id + 1000); + blob.setObjectKey(objectKey); + blob.setContentType(contentType); + blob.setSize(5L); + file.setBlob(blob); + return file; + } + + private Map readZipEntries(byte[] archiveBytes) throws Exception { + Map entries = new LinkedHashMap<>(); + try (ZipInputStream zipInputStream = new ZipInputStream( + new ByteArrayInputStream(archiveBytes), StandardCharsets.UTF_8)) { + var entry = zipInputStream.getNextEntry(); + while (entry != null) { + entries.put(entry.getName(), entry.isDirectory() ? "" : new String(zipInputStream.readAllBytes(), StandardCharsets.UTF_8)); + entry = zipInputStream.getNextEntry(); + } + } + return entries; + } + + private byte[] buildArchiveBytes(Map entries) throws Exception { + java.io.ByteArrayOutputStream outputStream = new java.io.ByteArrayOutputStream(); + try (java.util.zip.ZipOutputStream zipOutputStream = new java.util.zip.ZipOutputStream(outputStream, StandardCharsets.UTF_8)) { + for (Map.Entry entry : entries.entrySet()) { + zipOutputStream.putNextEntry(new java.util.zip.ZipEntry(entry.getKey())); + if (!entry.getKey().endsWith("/")) { + zipOutputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); + } + zipOutputStream.closeEntry(); + } + } + return outputStream.toByteArray(); + } +} diff --git a/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskServiceTest.java b/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskServiceTest.java new file mode 100644 index 0000000..0e87025 --- /dev/null +++ b/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskServiceTest.java @@ -0,0 +1,614 @@ +package com.yoyuzh.files.tasks; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.api.v2.ApiV2Exception; +import com.yoyuzh.auth.User; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class BackgroundTaskServiceTest { + + @Mock + private BackgroundTaskRepository backgroundTaskRepository; + + @Mock + private StoredFileRepository storedFileRepository; + + private BackgroundTaskService backgroundTaskService; + + @BeforeEach + void setUp() { + backgroundTaskService = new BackgroundTaskService(backgroundTaskRepository, storedFileRepository, new ObjectMapper()); + } + + @Test + void shouldRejectTaskCreationForForeignFile() { + User user = createUser(7L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(99L, 7L)).thenReturn(Optional.empty()); + + assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.ARCHIVE, + 99L, + "/docs/foreign.txt", + null + )).isInstanceOf(ApiV2Exception.class) + .hasMessage("file not found"); + } + + @Test + void shouldRejectTaskCreationForDeletedFile() { + User user = createUser(7L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(100L, 7L)).thenReturn(Optional.empty()); + + assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.ARCHIVE, + 100L, + "/docs/deleted.txt", + null + )).isInstanceOf(ApiV2Exception.class) + .hasMessage("file not found"); + } + + @Test + void shouldRejectTaskCreationWhenRequestedPathDoesNotMatchFile() { + User user = createUser(7L); + StoredFile file = createStoredFile(11L, user, "/docs", "real.txt", false, "text/plain", 3L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(file)); + + assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.ARCHIVE, + 11L, + "/docs/fake.txt", + null + )).isInstanceOf(ApiV2Exception.class) + .hasMessage("task path does not match file path"); + } + + @Test + void shouldRejectExtractTaskForDirectory() { + User user = createUser(7L); + StoredFile directory = createStoredFile(12L, user, "/", "bundle", true, null, 0L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(12L, 7L)).thenReturn(Optional.of(directory)); + + assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.EXTRACT, + 12L, + "/bundle", + null + )).isInstanceOf(ApiV2Exception.class) + .hasMessage("task target type is not supported"); + } + + @Test + void shouldRejectExtractTaskForNonZipCompatibleArchive() { + User user = createUser(7L); + StoredFile archive = createStoredFile(17L, user, "/docs", "backup.7z", false, "application/x-7z-compressed", 64L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(17L, 7L)).thenReturn(Optional.of(archive)); + + assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.EXTRACT, + 17L, + "/docs/backup.7z", + null + )).isInstanceOf(ApiV2Exception.class) + .hasMessage("extract task only supports zip-compatible archives"); + } + + @Test + void shouldRejectMediaMetadataTaskForNonMediaFile() { + User user = createUser(7L); + StoredFile file = createStoredFile(13L, user, "/docs", "notes.txt", false, "text/plain", 9L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(13L, 7L)).thenReturn(Optional.of(file)); + + assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.MEDIA_META, + 13L, + "/docs/notes.txt", + null + )).isInstanceOf(ApiV2Exception.class) + .hasMessage("media metadata task only supports media files"); + } + + @Test + void shouldCreateTaskStateFromServerFilePath() { + User user = createUser(7L); + StoredFile file = createStoredFile(14L, user, "/docs", "photo.png", false, "image/png", 15L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(14L, 7L)).thenReturn(Optional.of(file)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask task = backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.MEDIA_META, + 14L, + "/docs/photo.png", + "media-1" + ); + + assertThat(task.getPublicStateJson()).contains("\"fileId\":14"); + assertThat(task.getPublicStateJson()).contains("\"path\":\"/docs/photo.png\""); + assertThat(task.getPublicStateJson()).contains("\"filename\":\"photo.png\""); + assertThat(task.getPublicStateJson()).contains("\"directory\":false"); + assertThat(task.getPublicStateJson()).contains("\"contentType\":\"image/png\""); + assertThat(task.getPublicStateJson()).contains("\"size\":15"); + assertThat(task.getPublicStateJson()).contains("\"phase\":\"queued\""); + assertThat(task.getPublicStateJson()).contains("\"attemptCount\":0"); + assertThat(task.getPublicStateJson()).contains("\"maxAttempts\":2"); + assertThat(task.getPrivateStateJson()).contains("\"taskType\":\"MEDIA_META\""); + } + + @Test + void shouldCreateArchiveTaskStateWithDerivedOutputTarget() { + User user = createUser(7L); + StoredFile directory = createStoredFile(15L, user, "/docs", "archive", true, null, 0L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(15L, 7L)).thenReturn(Optional.of(directory)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask task = backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.ARCHIVE, + 15L, + "/docs/archive", + "archive-1" + ); + + assertThat(task.getPublicStateJson()).contains("\"outputPath\":\"/docs\""); + assertThat(task.getPublicStateJson()).contains("\"outputFilename\":\"archive.zip\""); + assertThat(task.getPublicStateJson()).contains("\"maxAttempts\":4"); + assertThat(task.getPrivateStateJson()).contains("\"outputPath\":\"/docs\""); + assertThat(task.getPrivateStateJson()).contains("\"outputFilename\":\"archive.zip\""); + } + + @Test + void shouldCreateExtractTaskStateWithDerivedOutputTarget() { + User user = createUser(7L); + StoredFile archive = createStoredFile(16L, user, "/docs", "extract.zip", false, "application/zip", 32L); + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(16L, 7L)).thenReturn(Optional.of(archive)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask task = backgroundTaskService.createQueuedFileTask( + user, + BackgroundTaskType.EXTRACT, + 16L, + "/docs/extract.zip", + "extract-1" + ); + + assertThat(task.getPublicStateJson()).contains("\"outputPath\":\"/docs\""); + assertThat(task.getPublicStateJson()).contains("\"outputDirectoryName\":\"extract\""); + assertThat(task.getPublicStateJson()).contains("\"maxAttempts\":3"); + assertThat(task.getPrivateStateJson()).contains("\"outputPath\":\"/docs\""); + assertThat(task.getPrivateStateJson()).contains("\"outputDirectoryName\":\"extract\""); + } + + @Test + void shouldClaimQueuedTaskOnlyWhenRepositoryTransitionSucceeds() { + BackgroundTask task = createTask(1L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING); + when(backgroundTaskRepository.claimQueuedTask( + eq(1L), + eq(BackgroundTaskStatus.QUEUED), + eq(BackgroundTaskStatus.RUNNING), + eq("worker-a"), + any(), + any(), + any() + )).thenReturn(1); + when(backgroundTaskRepository.findById(1L)).thenReturn(Optional.of(task)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + Optional result = backgroundTaskService.claimQueuedTask(1L, "worker-a", 120L); + + assertThat(result).containsSame(task); + assertThat(result.orElseThrow().getLeaseOwner()).isEqualTo("worker-a"); + assertThat(result.orElseThrow().getLeaseExpiresAt()).isNotNull(); + assertThat(result.orElseThrow().getHeartbeatAt()).isNotNull(); + assertThat(result.orElseThrow().getPublicStateJson()).contains("\"phase\":\"running\""); + assertThat(result.orElseThrow().getPublicStateJson()).contains("\"attemptCount\":1"); + assertThat(result.orElseThrow().getPublicStateJson()).contains("\"maxAttempts\":4"); + assertThat(result.orElseThrow().getPublicStateJson()).contains("\"workerOwner\":\"worker-a\""); + assertThat(result.orElseThrow().getPublicStateJson()).contains("\"heartbeatAt\":"); + assertThat(result.orElseThrow().getPublicStateJson()).contains("\"leaseExpiresAt\":"); + } + + @Test + void shouldNotClaimTaskWhenRepositoryTransitionWasSkipped() { + when(backgroundTaskRepository.claimQueuedTask( + eq(2L), + eq(BackgroundTaskStatus.QUEUED), + eq(BackgroundTaskStatus.RUNNING), + eq("worker-a"), + any(), + any(), + any() + )).thenReturn(0); + + Optional result = backgroundTaskService.claimQueuedTask(2L, "worker-a", 120L); + + assertThat(result).isEmpty(); + } + + @Test + void shouldCompleteRunningWorkerTaskAndMergePublicState() { + BackgroundTask task = createTask(3L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING); + task.setLeaseOwner("worker-a"); + task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120)); + task.setHeartbeatAt(java.time.LocalDateTime.now()); + task.setPublicStateJson("{\"fileId\":11,\"phase\":\"archiving\"}"); + when(backgroundTaskRepository.refreshRunningTaskLease( + eq(3L), + eq(BackgroundTaskStatus.RUNNING), + eq("worker-a"), + any(), + any(), + any() + )).thenReturn(1); + when(backgroundTaskRepository.findById(3L)).thenReturn(Optional.of(task)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask result = backgroundTaskService.markWorkerTaskCompleted(3L, "worker-a", Map.of("worker", "noop"), 120L); + + assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED); + assertThat(result.getFinishedAt()).isNotNull(); + assertThat(result.getErrorMessage()).isNull(); + assertThat(result.getLeaseOwner()).isNull(); + assertThat(result.getLeaseExpiresAt()).isNull(); + assertThat(result.getHeartbeatAt()).isNull(); + assertThat(result.getPublicStateJson()).contains("\"fileId\":11"); + assertThat(result.getPublicStateJson()).contains("\"worker\":\"noop\""); + assertThat(result.getPublicStateJson()).contains("\"phase\":\"completed\""); + assertThat(result.getPublicStateJson()).doesNotContain("workerOwner"); + assertThat(result.getPublicStateJson()).doesNotContain("leaseExpiresAt"); + } + + @Test + void shouldMergeWorkerProgressStateWhileTaskIsRunning() { + BackgroundTask task = createTask(7L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING); + task.setLeaseOwner("worker-a"); + task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120)); + task.setHeartbeatAt(java.time.LocalDateTime.now()); + task.setPublicStateJson("{\"fileId\":11,\"phase\":\"running\"}"); + when(backgroundTaskRepository.refreshRunningTaskLease( + eq(7L), + eq(BackgroundTaskStatus.RUNNING), + eq("worker-a"), + any(), + any(), + any() + )).thenReturn(1); + when(backgroundTaskRepository.findById(7L)).thenReturn(Optional.of(task)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask result = backgroundTaskService.markWorkerTaskProgress( + 7L, + "worker-a", + Map.of("phase", "extracting", "progressPercent", 50), + 120L + ); + + assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.RUNNING); + assertThat(result.getPublicStateJson()).contains("\"phase\":\"extracting\""); + assertThat(result.getPublicStateJson()).contains("\"progressPercent\":50"); + assertThat(result.getPublicStateJson()).contains("\"workerOwner\":\"worker-a\""); + assertThat(result.getPublicStateJson()).contains("\"heartbeatAt\":"); + assertThat(result.getPublicStateJson()).contains("\"leaseExpiresAt\":"); + } + + @Test + void shouldRecordTerminalWorkerFailureMessageWhenFailureIsNotRetryable() { + BackgroundTask task = createTask(4L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING); + task.setLeaseOwner("worker-a"); + task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120)); + task.setHeartbeatAt(java.time.LocalDateTime.now()); + task.setPublicStateJson("{\"fileId\":11,\"phase\":\"extracting\"}"); + when(backgroundTaskRepository.refreshRunningTaskLease( + eq(4L), + eq(BackgroundTaskStatus.RUNNING), + eq("worker-a"), + any(), + any(), + any() + )).thenReturn(1); + when(backgroundTaskRepository.findById(4L)).thenReturn(Optional.of(task)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask result = backgroundTaskService.markWorkerTaskFailed( + 4L, + "worker-a", + "media parser unavailable", + BackgroundTaskFailureCategory.DATA_STATE, + 120L + ); + + assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED); + assertThat(result.getFinishedAt()).isNotNull(); + assertThat(result.getErrorMessage()).isEqualTo("media parser unavailable"); + assertThat(result.getLeaseOwner()).isNull(); + assertThat(result.getLeaseExpiresAt()).isNull(); + assertThat(result.getHeartbeatAt()).isNull(); + assertThat(result.getPublicStateJson()).contains("\"phase\":\"failed\""); + assertThat(result.getPublicStateJson()).contains("\"attemptCount\":1"); + assertThat(result.getPublicStateJson()).contains("\"maxAttempts\":3"); + assertThat(result.getPublicStateJson()).contains("\"failureCategory\":\"DATA_STATE\""); + assertThat(result.getPublicStateJson()).doesNotContain("retryScheduled"); + } + + @Test + void shouldRequeueRetryableWorkerFailureWithBackoff() { + BackgroundTask task = createTask(14L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING); + task.setLeaseOwner("worker-a"); + task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120)); + task.setHeartbeatAt(java.time.LocalDateTime.now()); + task.setPublicStateJson("{\"fileId\":11,\"phase\":\"extracting\"}"); + when(backgroundTaskRepository.refreshRunningTaskLease( + eq(14L), + eq(BackgroundTaskStatus.RUNNING), + eq("worker-a"), + any(), + any(), + any() + )).thenReturn(1); + when(backgroundTaskRepository.findById(14L)).thenReturn(Optional.of(task)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask result = backgroundTaskService.markWorkerTaskFailed( + 14L, + "worker-a", + "storage timeout", + BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE, + 120L + ); + + assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED); + assertThat(result.getFinishedAt()).isNull(); + assertThat(result.getErrorMessage()).isNull(); + assertThat(result.getNextRunAt()).isNotNull(); + assertThat(result.getLeaseOwner()).isNull(); + assertThat(result.getLeaseExpiresAt()).isNull(); + assertThat(result.getHeartbeatAt()).isNull(); + assertThat(result.getPublicStateJson()).contains("\"phase\":\"queued\""); + assertThat(result.getPublicStateJson()).contains("\"retryScheduled\":true"); + assertThat(result.getPublicStateJson()).contains("\"retryDelaySeconds\":30"); + assertThat(result.getPublicStateJson()).contains("\"failureCategory\":\"TRANSIENT_INFRASTRUCTURE\""); + assertThat(result.getPublicStateJson()).contains("\"lastFailureMessage\":\"storage timeout\""); + } + + @Test + void shouldUseLongerBackoffForRateLimitedFailures() { + BackgroundTask task = createTask(18L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING); + task.setLeaseOwner("worker-a"); + task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120)); + task.setHeartbeatAt(java.time.LocalDateTime.now()); + task.setPublicStateJson("{\"fileId\":11,\"phase\":\"archiving\"}"); + when(backgroundTaskRepository.refreshRunningTaskLease( + eq(18L), + eq(BackgroundTaskStatus.RUNNING), + eq("worker-a"), + any(), + any(), + any() + )).thenReturn(1); + when(backgroundTaskRepository.findById(18L)).thenReturn(Optional.of(task)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask result = backgroundTaskService.markWorkerTaskFailed( + 18L, + "worker-a", + "429 too many requests", + BackgroundTaskFailureCategory.RATE_LIMITED, + 120L + ); + + assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED); + assertThat(result.getPublicStateJson()).contains("\"retryDelaySeconds\":120"); + assertThat(result.getPublicStateJson()).contains("\"failureCategory\":\"RATE_LIMITED\""); + } + + @Test + void shouldFailTerminallyWhenRetryableFailureExhaustsAttempts() { + BackgroundTask task = createTask(15L, BackgroundTaskType.MEDIA_META, BackgroundTaskStatus.RUNNING); + task.setAttemptCount(2); + task.setMaxAttempts(2); + task.setLeaseOwner("worker-a"); + task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120)); + task.setHeartbeatAt(java.time.LocalDateTime.now()); + task.setPublicStateJson("{\"fileId\":11,\"phase\":\"extracting-metadata\",\"attemptCount\":2,\"maxAttempts\":2}"); + when(backgroundTaskRepository.refreshRunningTaskLease( + eq(15L), + eq(BackgroundTaskStatus.RUNNING), + eq("worker-a"), + any(), + any(), + any() + )).thenReturn(1); + when(backgroundTaskRepository.findById(15L)).thenReturn(Optional.of(task)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask result = backgroundTaskService.markWorkerTaskFailed( + 15L, + "worker-a", + "storage timeout", + BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE, + 120L + ); + + assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED); + assertThat(result.getFinishedAt()).isNotNull(); + assertThat(result.getErrorMessage()).isEqualTo("storage timeout"); + assertThat(result.getPublicStateJson()).contains("\"phase\":\"failed\""); + assertThat(result.getPublicStateJson()).contains("\"failureCategory\":\"TRANSIENT_INFRASTRUCTURE\""); + assertThat(result.getPublicStateJson()).doesNotContain("retryScheduled"); + } + + @Test + void shouldRetryFailedTaskAndResetPublicState() { + User user = createUser(7L); + BackgroundTask task = createTask(8L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.FAILED); + task.setPublicStateJson(""" + {"fileId":11,"path":"/docs/extract.zip","phase":"failed","worker":"extract","processedFileCount":1,"totalFileCount":2} + """); + task.setPrivateStateJson(""" + {"fileId":11,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"} + """); + task.setFinishedAt(java.time.LocalDateTime.now()); + task.setErrorMessage("extract task only supports zip-compatible archives"); + when(backgroundTaskRepository.findByIdAndUserId(8L, 7L)).thenReturn(Optional.of(task)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + BackgroundTask result = backgroundTaskService.retryOwnedTask(user, 8L); + + assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED); + assertThat(result.getFinishedAt()).isNull(); + assertThat(result.getErrorMessage()).isNull(); + assertThat(result.getPublicStateJson()).contains("\"phase\":\"queued\""); + assertThat(result.getPublicStateJson()).contains("\"outputPath\":\"/docs\""); + assertThat(result.getPublicStateJson()).contains("\"outputDirectoryName\":\"extract\""); + assertThat(result.getPublicStateJson()).contains("\"attemptCount\":0"); + assertThat(result.getPublicStateJson()).contains("\"maxAttempts\":3"); + assertThat(result.getPublicStateJson()).doesNotContain("taskType"); + assertThat(result.getPublicStateJson()).doesNotContain("worker"); + assertThat(result.getPublicStateJson()).doesNotContain("processedFileCount"); + assertThat(result.getPublicStateJson()).doesNotContain("totalFileCount"); + } + + @Test + void shouldRejectRetryForNonFailedTask() { + User user = createUser(7L); + BackgroundTask task = createTask(9L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.COMPLETED); + when(backgroundTaskRepository.findByIdAndUserId(9L, 7L)).thenReturn(Optional.of(task)); + + assertThatThrownBy(() -> backgroundTaskService.retryOwnedTask(user, 9L)) + .isInstanceOf(ApiV2Exception.class) + .hasMessage("only failed tasks can be retried"); + } + + @Test + void shouldRequeueOnlyExpiredRunningTasksOnStartup() { + BackgroundTask expired = createTask(10L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING); + expired.setLeaseOwner("worker-stale"); + expired.setLeaseExpiresAt(java.time.LocalDateTime.now().minusSeconds(5)); + expired.setHeartbeatAt(java.time.LocalDateTime.now().minusSeconds(10)); + expired.setPublicStateJson(""" + {"fileId":11,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-stale"} + """); + expired.setPrivateStateJson(""" + {"fileId":11,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"} + """); + expired.setFinishedAt(java.time.LocalDateTime.now()); + expired.setErrorMessage("partial failure"); + BackgroundTask fresh = createTask(11L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING); + fresh.setLeaseOwner("worker-live"); + fresh.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(300)); + fresh.setHeartbeatAt(java.time.LocalDateTime.now()); + when(backgroundTaskRepository.findExpiredRunningTaskIds(eq(BackgroundTaskStatus.RUNNING), any(), any())) + .thenReturn(List.of(10L)); + when(backgroundTaskRepository.requeueExpiredRunningTask( + eq(10L), + eq(BackgroundTaskStatus.RUNNING), + eq(BackgroundTaskStatus.QUEUED), + any(), + any() + )).thenReturn(1); + when(backgroundTaskRepository.findById(10L)).thenReturn(Optional.of(expired)); + when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + int recovered = backgroundTaskService.requeueExpiredRunningTasks(); + + assertThat(recovered).isEqualTo(1); + assertThat(expired.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED); + assertThat(expired.getFinishedAt()).isNull(); + assertThat(expired.getErrorMessage()).isNull(); + assertThat(expired.getLeaseOwner()).isNull(); + assertThat(expired.getLeaseExpiresAt()).isNull(); + assertThat(expired.getHeartbeatAt()).isNull(); + assertThat(expired.getPublicStateJson()).contains("\"phase\":\"queued\""); + assertThat(expired.getPublicStateJson()).contains("\"attemptCount\":1"); + assertThat(expired.getPublicStateJson()).contains("\"maxAttempts\":3"); + assertThat(expired.getPublicStateJson()).contains("\"outputPath\":\"/docs\""); + assertThat(expired.getPublicStateJson()).contains("\"outputDirectoryName\":\"extract\""); + assertThat(expired.getPublicStateJson()).doesNotContain("worker"); + assertThat(expired.getPublicStateJson()).doesNotContain("taskType"); + assertThat(expired.getPublicStateJson()).doesNotContain("workerOwner"); + assertThat(fresh.getStatus()).isEqualTo(BackgroundTaskStatus.RUNNING); + } + + @Test + void shouldFindQueuedTaskIdsInCreatedOrderLimit() { + when(backgroundTaskRepository.findReadyTaskIdsByStatusOrder(eq(BackgroundTaskStatus.QUEUED), any(), any())) + .thenReturn(List.of(5L, 6L)); + + List result = backgroundTaskService.findQueuedTaskIds(2); + + assertThat(result).containsExactly(5L, 6L); + } + + @Test + void shouldReturnEmptyTaskIdsWhenLimitIsNonPositive() { + List result = backgroundTaskService.findQueuedTaskIds(0); + assertThat(result).isEmpty(); + } + + private BackgroundTask createTask(Long id, BackgroundTaskType type, BackgroundTaskStatus status) { + BackgroundTask task = new BackgroundTask(); + task.setId(id); + task.setType(type); + task.setStatus(status); + task.setUserId(7L); + task.setAttemptCount(status == BackgroundTaskStatus.RUNNING ? 1 : 0); + task.setMaxAttempts(switch (type) { + case ARCHIVE -> 4; + case EXTRACT -> 3; + case MEDIA_META -> 2; + default -> 1; + }); + task.setPublicStateJson("{}"); + task.setPrivateStateJson("{}"); + return task; + } + + private User createUser(Long id) { + User user = new User(); + user.setId(id); + user.setUsername("alice"); + return user; + } + + private StoredFile createStoredFile(Long id, + User user, + String path, + String filename, + boolean directory, + String contentType, + Long size) { + StoredFile file = new StoredFile(); + file.setId(id); + file.setUser(user); + file.setPath(path); + file.setFilename(filename); + file.setDirectory(directory); + file.setContentType(contentType); + file.setSize(size); + return file; + } +} diff --git a/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskWorkerTest.java b/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskWorkerTest.java new file mode 100644 index 0000000..2a61bec --- /dev/null +++ b/backend/src/test/java/com/yoyuzh/files/tasks/BackgroundTaskWorkerTest.java @@ -0,0 +1,139 @@ +package com.yoyuzh.files.tasks; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class BackgroundTaskWorkerTest { + + @Mock + private BackgroundTaskService backgroundTaskService; + @Mock + private BackgroundTaskHandler backgroundTaskHandler; + + private BackgroundTaskWorker backgroundTaskWorker; + + @BeforeEach + void setUp() { + backgroundTaskWorker = new BackgroundTaskWorker(backgroundTaskService, List.of(backgroundTaskHandler)); + } + + @Test + void shouldClaimAndCompleteQueuedTaskThroughNoopHandler() { + BackgroundTask task = createTask(1L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING); + when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L)); + when(backgroundTaskService.claimQueuedTask(eq(1L), anyString(), anyLong())).thenReturn(Optional.of(task)); + when(backgroundTaskHandler.supports(BackgroundTaskType.ARCHIVE)).thenReturn(true); + when(backgroundTaskHandler.handle(eq(task), any(BackgroundTaskProgressReporter.class))) + .thenReturn(new BackgroundTaskHandlerResult(Map.of("worker", "noop"))); + + int processedCount = backgroundTaskWorker.processQueuedTasks(5); + + assertThat(processedCount).isEqualTo(1); + verify(backgroundTaskService).markWorkerTaskProgress(eq(1L), anyString(), eq(Map.of("phase", "archiving")), anyLong()); + verify(backgroundTaskHandler).handle(eq(task), any(BackgroundTaskProgressReporter.class)); + verify(backgroundTaskService).markWorkerTaskCompleted(eq(1L), anyString(), eq(Map.of("worker", "noop")), anyLong()); + } + + @Test + void shouldSkipTaskThatWasNotClaimed() { + when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L)); + when(backgroundTaskService.claimQueuedTask(eq(1L), anyString(), anyLong())).thenReturn(Optional.empty()); + + int processedCount = backgroundTaskWorker.processQueuedTasks(5); + + assertThat(processedCount).isZero(); + verify(backgroundTaskHandler, never()).handle(any(BackgroundTask.class), any(BackgroundTaskProgressReporter.class)); + } + + @Test + void shouldMarkTaskFailedWhenHandlerThrows() { + BackgroundTask task = createTask(2L, BackgroundTaskType.MEDIA_META, BackgroundTaskStatus.RUNNING); + when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(2L)); + when(backgroundTaskService.claimQueuedTask(eq(2L), anyString(), anyLong())).thenReturn(Optional.of(task)); + when(backgroundTaskHandler.supports(BackgroundTaskType.MEDIA_META)).thenReturn(true); + when(backgroundTaskHandler.handle(eq(task), any(BackgroundTaskProgressReporter.class))) + .thenThrow(new IllegalStateException("media parser unavailable")); + + int processedCount = backgroundTaskWorker.processQueuedTasks(5); + + assertThat(processedCount).isEqualTo(1); + verify(backgroundTaskService).markWorkerTaskProgress(eq(2L), anyString(), eq(Map.of("phase", "extracting-metadata")), anyLong()); + verify(backgroundTaskService).markWorkerTaskFailed( + eq(2L), + anyString(), + eq("media parser unavailable"), + eq(BackgroundTaskFailureCategory.DATA_STATE), + anyLong() + ); + } + + @Test + void shouldAutoRetryUnexpectedWorkerFailure() { + BackgroundTask task = createTask(3L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING); + when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(3L)); + when(backgroundTaskService.claimQueuedTask(eq(3L), anyString(), anyLong())).thenReturn(Optional.of(task)); + when(backgroundTaskHandler.supports(BackgroundTaskType.ARCHIVE)).thenReturn(true); + when(backgroundTaskHandler.handle(eq(task), any(BackgroundTaskProgressReporter.class))) + .thenThrow(new RuntimeException("storage timeout")); + + int processedCount = backgroundTaskWorker.processQueuedTasks(5); + + assertThat(processedCount).isEqualTo(1); + verify(backgroundTaskService).markWorkerTaskFailed( + eq(3L), + anyString(), + eq("storage timeout"), + eq(BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE), + anyLong() + ); + } + + @Test + void shouldClassifyRateLimitedFailureSeparately() { + BackgroundTask task = createTask(4L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING); + when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(4L)); + when(backgroundTaskService.claimQueuedTask(eq(4L), anyString(), anyLong())).thenReturn(Optional.of(task)); + when(backgroundTaskHandler.supports(BackgroundTaskType.EXTRACT)).thenReturn(true); + when(backgroundTaskHandler.handle(eq(task), any(BackgroundTaskProgressReporter.class))) + .thenThrow(new RuntimeException("429 Too Many Requests")); + + int processedCount = backgroundTaskWorker.processQueuedTasks(5); + + assertThat(processedCount).isEqualTo(1); + verify(backgroundTaskService).markWorkerTaskFailed( + eq(4L), + anyString(), + eq("429 Too Many Requests"), + eq(BackgroundTaskFailureCategory.RATE_LIMITED), + anyLong() + ); + } + + private BackgroundTask createTask(Long id, BackgroundTaskType type, BackgroundTaskStatus status) { + BackgroundTask task = new BackgroundTask(); + task.setId(id); + task.setType(type); + task.setStatus(status); + task.setUserId(7L); + task.setPublicStateJson("{}"); + task.setPrivateStateJson("{}"); + return task; + } +} diff --git a/backend/src/test/java/com/yoyuzh/files/tasks/ExtractBackgroundTaskHandlerTest.java b/backend/src/test/java/com/yoyuzh/files/tasks/ExtractBackgroundTaskHandlerTest.java new file mode 100644 index 0000000..16636b9 --- /dev/null +++ b/backend/src/test/java/com/yoyuzh/files/tasks/ExtractBackgroundTaskHandlerTest.java @@ -0,0 +1,193 @@ +package com.yoyuzh.files.tasks; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.auth.User; +import com.yoyuzh.auth.UserRepository; +import com.yoyuzh.common.BusinessException; +import com.yoyuzh.common.ErrorCode; +import com.yoyuzh.files.core.FileBlob; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ExtractBackgroundTaskHandlerTest { + + @Mock + private StoredFileRepository storedFileRepository; + @Mock + private UserRepository userRepository; + @Mock + private FileService fileService; + + private ExtractBackgroundTaskHandler handler; + + @BeforeEach + void setUp() { + handler = new ExtractBackgroundTaskHandler( + storedFileRepository, + userRepository, + fileService, + new ObjectMapper() + ); + } + + @Test + void shouldExtractArchivedDirectoryIntoSiblingFolder() throws Exception { + User user = createUser(7L); + StoredFile archive = createArchiveFile(11L, user, "/docs", "archive.zip", "application/zip", "blobs/archive.zip"); + + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(archive)); + when(userRepository.findById(7L)).thenReturn(Optional.of(user)); + when(fileService.readZipCompatibleArchive(archive)).thenReturn(new FileService.ZipCompatibleArchive( + java.util.List.of( + new FileService.ZipCompatibleArchiveEntry("archive", true, new byte[0]), + new FileService.ZipCompatibleArchiveEntry("archive/nested", true, new byte[0]), + new FileService.ZipCompatibleArchiveEntry("archive/notes.txt", false, "hello".getBytes(StandardCharsets.UTF_8)), + new FileService.ZipCompatibleArchiveEntry("archive/nested/todo.txt", false, "world".getBytes(StandardCharsets.UTF_8)) + ), + "archive" + )); + + BackgroundTaskHandlerResult result = handler.handle(createExtractTask(11L, 7L, "archive")); + + verify(fileService).importExternalFilesAtomically( + eq(user), + eq(java.util.List.of("/docs/archive", "/docs/archive/nested")), + argThat(files -> files.size() == 2 + && files.stream().anyMatch(file -> "/docs/archive".equals(file.path()) + && "notes.txt".equals(file.filename()) + && "text/plain".equals(file.contentType()) + && java.util.Arrays.equals("hello".getBytes(StandardCharsets.UTF_8), file.content())) + && files.stream().anyMatch(file -> "/docs/archive/nested".equals(file.path()) + && "todo.txt".equals(file.filename()) + && "text/plain".equals(file.contentType()) + && java.util.Arrays.equals("world".getBytes(StandardCharsets.UTF_8), file.content()))), + any() + ); + assertThat(result.publicStatePatch()).containsEntry("worker", "extract"); + assertThat(result.publicStatePatch()).containsEntry("extractedPath", "/docs/archive"); + assertThat(result.publicStatePatch()).containsEntry("extractedFileCount", 2); + assertThat(result.publicStatePatch()).containsEntry("extractedDirectoryCount", 2); + assertThat(result.publicStatePatch()).containsEntry("processedFileCount", 2); + assertThat(result.publicStatePatch()).containsEntry("totalFileCount", 2); + assertThat(result.publicStatePatch()).containsEntry("processedDirectoryCount", 2); + assertThat(result.publicStatePatch()).containsEntry("totalDirectoryCount", 2); + } + + @Test + void shouldExtractSingleArchivedFileBackIntoParentPath() throws Exception { + User user = createUser(7L); + StoredFile archive = createArchiveFile(21L, user, "/docs", "notes.txt.zip", "application/zip", "blobs/notes.txt.zip"); + + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(21L, 7L)).thenReturn(Optional.of(archive)); + when(userRepository.findById(7L)).thenReturn(Optional.of(user)); + when(fileService.readZipCompatibleArchive(archive)).thenReturn(new FileService.ZipCompatibleArchive( + java.util.List.of( + new FileService.ZipCompatibleArchiveEntry("notes.txt", false, "hello".getBytes(StandardCharsets.UTF_8)) + ), + null + )); + + BackgroundTaskHandlerResult result = handler.handle(createExtractTask(21L, 7L, "notes.txt")); + + verify(fileService).importExternalFilesAtomically( + eq(user), + eq(java.util.List.of()), + argThat(files -> files.size() == 1 + && "/docs".equals(files.get(0).path()) + && "notes.txt".equals(files.get(0).filename()) + && "text/plain".equals(files.get(0).contentType()) + && java.util.Arrays.equals("hello".getBytes(StandardCharsets.UTF_8), files.get(0).content())), + any() + ); + assertThat(result.publicStatePatch()).containsEntry("worker", "extract"); + assertThat(result.publicStatePatch()).containsEntry("extractedPath", "/docs"); + assertThat(result.publicStatePatch()).containsEntry("extractedFileCount", 1); + assertThat(result.publicStatePatch()).containsEntry("extractedDirectoryCount", 0); + assertThat(result.publicStatePatch()).containsEntry("processedFileCount", 1); + assertThat(result.publicStatePatch()).containsEntry("totalFileCount", 1); + assertThat(result.publicStatePatch()).containsEntry("processedDirectoryCount", 0); + assertThat(result.publicStatePatch()).containsEntry("totalDirectoryCount", 0); + } + + @Test + void shouldRejectNonZipCompatibleArchiveContent() { + User user = createUser(7L); + StoredFile archive = createArchiveFile(31L, user, "/docs", "backup.7z", "application/x-7z-compressed", "blobs/backup.7z"); + + when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(31L, 7L)).thenReturn(Optional.of(archive)); + when(userRepository.findById(7L)).thenReturn(Optional.of(user)); + when(fileService.readZipCompatibleArchive(archive)) + .thenThrow(new BusinessException(ErrorCode.UNKNOWN, "压缩包读取失败")); + + assertThatThrownBy(() -> handler.handle(createExtractTask(31L, 7L, "backup"))) + .isInstanceOf(IllegalStateException.class) + .hasMessage("extract task only supports zip-compatible archives"); + + verify(fileService, never()).importExternalFilesAtomically(any(), any(), any(), any()); + } + + private BackgroundTask createExtractTask(Long fileId, Long userId, String outputDirectoryName) { + BackgroundTask task = new BackgroundTask(); + task.setId(401L); + task.setType(BackgroundTaskType.EXTRACT); + task.setStatus(BackgroundTaskStatus.RUNNING); + task.setUserId(userId); + task.setPublicStateJson(""" + {"fileId":%d,"outputPath":"/docs","outputDirectoryName":"%s"} + """.formatted(fileId, outputDirectoryName)); + task.setPrivateStateJson(""" + {"fileId":%d,"taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"%s"} + """.formatted(fileId, outputDirectoryName)); + return task; + } + + private User createUser(Long id) { + User user = new User(); + user.setId(id); + user.setUsername("alice"); + return user; + } + + private StoredFile createArchiveFile(Long id, + User user, + String path, + String filename, + String contentType, + String objectKey) { + StoredFile file = new StoredFile(); + file.setId(id); + file.setUser(user); + file.setPath(path); + file.setFilename(filename); + file.setDirectory(false); + file.setContentType(contentType); + file.setSize(12L); + FileBlob blob = new FileBlob(); + blob.setId(id + 1000); + blob.setObjectKey(objectKey); + blob.setContentType(contentType); + blob.setSize(12L); + file.setBlob(blob); + return file; + } +} diff --git a/backend/src/test/java/com/yoyuzh/files/MediaMetadataBackgroundTaskHandlerTest.java b/backend/src/test/java/com/yoyuzh/files/tasks/MediaMetadataBackgroundTaskHandlerTest.java similarity index 95% rename from backend/src/test/java/com/yoyuzh/files/MediaMetadataBackgroundTaskHandlerTest.java rename to backend/src/test/java/com/yoyuzh/files/tasks/MediaMetadataBackgroundTaskHandlerTest.java index 5e772e6..a0aa533 100644 --- a/backend/src/test/java/com/yoyuzh/files/MediaMetadataBackgroundTaskHandlerTest.java +++ b/backend/src/test/java/com/yoyuzh/files/tasks/MediaMetadataBackgroundTaskHandlerTest.java @@ -1,6 +1,11 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.tasks; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.files.core.FileBlob; +import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.search.FileMetadata; +import com.yoyuzh.files.search.FileMetadataRepository; import com.yoyuzh.files.storage.FileContentStorage; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -122,10 +127,10 @@ class MediaMetadataBackgroundTaskHandlerTest { } @Test - void shouldKeepNoopHandlerLimitedToArchiveAndExtract() { + void shouldKeepNoopHandlerOutOfArchiveExtractAndMediaMetadata() { NoopBackgroundTaskHandler noop = new NoopBackgroundTaskHandler(); - assertThat(noop.supports(BackgroundTaskType.ARCHIVE)).isTrue(); - assertThat(noop.supports(BackgroundTaskType.EXTRACT)).isTrue(); + assertThat(noop.supports(BackgroundTaskType.ARCHIVE)).isFalse(); + assertThat(noop.supports(BackgroundTaskType.EXTRACT)).isFalse(); assertThat(noop.supports(BackgroundTaskType.MEDIA_META)).isFalse(); } diff --git a/backend/src/test/java/com/yoyuzh/files/UploadSessionServiceTest.java b/backend/src/test/java/com/yoyuzh/files/upload/UploadSessionServiceTest.java similarity index 77% rename from backend/src/test/java/com/yoyuzh/files/UploadSessionServiceTest.java rename to backend/src/test/java/com/yoyuzh/files/upload/UploadSessionServiceTest.java index 0a0c954..c1059df 100644 --- a/backend/src/test/java/com/yoyuzh/files/UploadSessionServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/files/upload/UploadSessionServiceTest.java @@ -1,9 +1,16 @@ -package com.yoyuzh.files; +package com.yoyuzh.files.upload; import com.yoyuzh.auth.User; import com.yoyuzh.common.BusinessException; import com.yoyuzh.config.FileStorageProperties; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.policy.StoragePolicy; +import com.yoyuzh.files.policy.StoragePolicyCapabilities; +import com.yoyuzh.files.policy.StoragePolicyService; +import com.yoyuzh.files.policy.StoragePolicyType; import com.yoyuzh.files.storage.FileContentStorage; +import com.yoyuzh.files.storage.PreparedUpload; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -15,6 +22,7 @@ import java.time.Clock; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.Map; import java.util.List; import java.util.Optional; @@ -61,7 +69,20 @@ class UploadSessionServiceTest { void shouldCreateUploadSessionWithoutChangingLegacyUploadPath() { User user = createUser(7L); when(storedFileRepository.existsByUserIdAndPathAndFilename(7L, "/docs", "movie.mp4")).thenReturn(false); - when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy()); + StoragePolicy policy = createDefaultStoragePolicy(); + when(storagePolicyService.ensureDefaultPolicy()).thenReturn(policy); + when(storagePolicyService.readCapabilities(policy)).thenReturn(new StoragePolicyCapabilities( + true, + true, + true, + true, + false, + true, + true, + false, + 500L * 1024 * 1024 + )); + when(fileContentStorage.createMultipartUpload(any(), eq("video/mp4"))).thenReturn("upload-123"); when(uploadSessionRepository.save(any(UploadSession.class))).thenAnswer(invocation -> { UploadSession session = invocation.getArgument(0); session.setId(100L); @@ -75,6 +96,7 @@ class UploadSessionServiceTest { assertThat(session.getSessionId()).isNotBlank(); assertThat(session.getObjectKey()).startsWith("blobs/"); + assertThat(session.getMultipartUploadId()).isEqualTo("upload-123"); assertThat(session.getStatus()).isEqualTo(UploadSessionStatus.CREATED); assertThat(session.getStoragePolicyId()).isEqualTo(42L); assertThat(session.getChunkSize()).isEqualTo(8L * 1024 * 1024); @@ -82,6 +104,36 @@ class UploadSessionServiceTest { assertThat(session.getExpiresAt()).isEqualTo(LocalDateTime.of(2026, 4, 9, 6, 0)); } + @Test + void shouldPrepareMultipartPartUploadForOwnedSession() { + User user = createUser(7L); + UploadSession session = createSession(user); + session.setMultipartUploadId("upload-123"); + session.setChunkCount(3); + session.setChunkSize(8L * 1024 * 1024); + session.setSize(20L * 1024 * 1024); + when(uploadSessionRepository.findBySessionIdAndUserId("session-1", 7L)) + .thenReturn(Optional.of(session)); + when(fileContentStorage.prepareMultipartPartUpload( + "blobs/session-1", + "upload-123", + 3, + "video/mp4", + 4L * 1024 * 1024 + )).thenReturn(new PreparedUpload( + true, + "https://upload.example.com/session-1/part-3", + "PUT", + Map.of("Content-Type", "video/mp4"), + "blobs/session-1" + )); + + PreparedUpload preparedUpload = uploadSessionService.prepareOwnedPartUpload(user, "session-1", 2); + + assertThat(preparedUpload.uploadUrl()).isEqualTo("https://upload.example.com/session-1/part-3"); + assertThat(preparedUpload.method()).isEqualTo("PUT"); + } + @Test void shouldOnlyReturnSessionOwnedByCurrentUser() { User user = createUser(7L); @@ -112,6 +164,15 @@ class UploadSessionServiceTest { void shouldCompleteOwnedSessionThroughLegacyFileCommitPath() { User user = createUser(7L); UploadSession session = createSession(user); + session.setMultipartUploadId("upload-123"); + session.setChunkCount(2); + session.setChunkSize(8L * 1024 * 1024); + session.setUploadedPartsJson(""" + [ + {"partIndex":0,"etag":"etag-1","size":8388608,"uploadedAt":"2026-04-08T06:00:00"}, + {"partIndex":1,"etag":"etag-2","size":12,"uploadedAt":"2026-04-08T06:01:00"} + ] + """); when(uploadSessionRepository.findBySessionIdAndUserId("session-1", 7L)) .thenReturn(Optional.of(session)); when(uploadSessionRepository.save(any(UploadSession.class))).thenAnswer(invocation -> invocation.getArgument(0)); @@ -120,6 +181,7 @@ class UploadSessionServiceTest { assertThat(result.getStatus()).isEqualTo(UploadSessionStatus.COMPLETED); assertThat(result.getUpdatedAt()).isEqualTo(LocalDateTime.of(2026, 4, 8, 6, 0)); + verify(fileContentStorage).completeMultipartUpload(eq("blobs/session-1"), eq("upload-123"), anyList()); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CompleteUploadRequest.class); verify(fileService).completeUpload(eq(user), requestCaptor.capture()); assertThat(requestCaptor.getValue().path()).isEqualTo("/docs"); @@ -196,6 +258,7 @@ class UploadSessionServiceTest { UploadSession session = createSession(user); session.setStatus(UploadSessionStatus.UPLOADING); session.setObjectKey("blobs/expired-session"); + session.setMultipartUploadId("upload-expired"); session.setExpiresAt(LocalDateTime.of(2026, 4, 8, 5, 0)); when(uploadSessionRepository.findByStatusInAndExpiresAtBefore(anyList(), eq(LocalDateTime.of(2026, 4, 8, 6, 0)))) .thenReturn(List.of(session)); @@ -205,7 +268,7 @@ class UploadSessionServiceTest { assertThat(expiredCount).isEqualTo(1); assertThat(session.getStatus()).isEqualTo(UploadSessionStatus.EXPIRED); assertThat(session.getUpdatedAt()).isEqualTo(LocalDateTime.of(2026, 4, 8, 6, 0)); - verify(fileContentStorage).deleteBlob("blobs/expired-session"); + verify(fileContentStorage).abortMultipartUpload("blobs/expired-session", "upload-expired"); verify(uploadSessionRepository).saveAll(List.of(session)); } @@ -238,6 +301,7 @@ class UploadSessionServiceTest { session.setContentType("video/mp4"); session.setSize(20L); session.setObjectKey("blobs/session-1"); + session.setMultipartUploadId(null); session.setChunkSize(8L * 1024 * 1024); session.setChunkCount(1); session.setUploadedPartsJson("[]"); diff --git a/docs/agents/unfinished-work.md b/docs/agents/unfinished-work.md index 427b7da..c2531cf 100644 --- a/docs/agents/unfinished-work.md +++ b/docs/agents/unfinished-work.md @@ -4,15 +4,35 @@ 本文只记录当前还没有完全收口的工作，方便后续窗口快速接上。新会话仍必须先读 `memory.md`、`docs/architecture.md`、`docs/api-reference.md`，再读本文。 +## 当前 files 后端结构 + +`backend/src/main/java/com/yoyuzh/files` 已完成一次只改结构、不改语义的包清理，当前按职责拆为： + +- `com.yoyuzh.files.core` +- `com.yoyuzh.files.upload` +- `com.yoyuzh.files.share` +- `com.yoyuzh.files.search` +- `com.yoyuzh.files.events` +- `com.yoyuzh.files.tasks` +- `com.yoyuzh.files.storage` +- `com.yoyuzh.files.policy` + +注意： + +- 旧的 API 路径、数据库表/字段、响应结构、前端调用方式都没改 +- 后续做 files 相关工作时，先按上面职责分区找类，不要再默认去平铺的 `com.yoyuzh.files.*` 根包下找 +- `FileService` 仍然是当前 files 子系统最重的协调类，后续若继续做结构优化，应优先评估它和 `FileController` 是否值得继续按用例拆分 + ## 当前 Git 状态 - 当前分支：`dev` - 已推送到：`gitea/dev` - 最近已推送提交：`977eb60 feat(files): add v2 task and metadata workflows` -- 当前未提交文件： - - `docs/superpowers/plans/2026-04-09-frontend-redesign-generation-spec.md` +- 当前 worktree 不是干净状态：除本次后端 multipart / 后台任务相关文件外，还混有前端重构中的改动与若干新增文件；提交前先用 `git status` 逐项确认，不要批量回滚 - 当前未跟踪但不要默认处理： - `.claude/` + - `docs/superpowers/plans/2026-04-09-multi-user-platform-upgrade-phase-2.md` + - 前端重构过程中新增的 `front/src/components/ui/*`、`front/src/mobile-pages/*`、`front/src/pages/files/*` 等文件 ## 已写好的前端重设计说明书 @@ -90,22 +110,24 @@ npm run test 不要在仓库根目录运行 `npm` 命令。 -### P1：阶段 6 后台任务继续真实化 +### P1：阶段 6 后台任务继续收口 当前状态： - `/api/v2/tasks/**` 后端骨架已落地 - worker 会领取 `QUEUED` 任务并切换状态 - `MEDIA_META` 已有最小真实 handler，会写入基础媒体 metadata 和图片宽高 -- `ARCHIVE` / `EXTRACT` 仍是 no-op handler +- `ARCHIVE` 已有真实 handler：会派生 `outputPath/outputFilename`，生成 zip 并回写同级目录 +- `EXTRACT` 已有真实 handler：当前只支持 zip-compatible 归档，会派生 `outputPath/outputDirectoryName`，剥离共享根目录，并在批量导入失败时清理已写入 blob +- `/api/v2/tasks/**` 已有最小 progress 字段：`publicStateJson.phase` 会经历 `queued -> running -> archiving/extracting/extracting-metadata -> completed/failed/cancelled` +- `ARCHIVE/EXTRACT` 还会暴露真实计数字段：`processedFileCount/totalFileCount`、`processedDirectoryCount/totalDirectoryCount` 与真实 `progressPercent` +- 后台任务已有按任务类型区分的自动重试：`ARCHIVE` 默认最多 4 次、`EXTRACT` 最多 3 次、`MEDIA_META` 最多 2 次；失败会归类为 `UNSUPPORTED_INPUT/DATA_STATE/TRANSIENT_INFRASTRUCTURE/RATE_LIMITED/UNKNOWN`，自动重试时会写入 `retryScheduled/nextRetryAt/retryDelaySeconds/lastFailureMessage/lastFailureAt/failureCategory` +- `/api/v2/tasks/{id}/retry` 已支持最小手动重试：仅 `FAILED` 任务可由当前用户重置回 `QUEUED`，并清空 `finishedAt/errorMessage` +- worker 已有 heartbeat 与多实例 lease：运行中会暴露 `workerOwner/heartbeatAt/leaseExpiresAt/startedAt`，服务启动和调度前都只回收 lease 已过期的 `RUNNING` 任务 +- `BackgroundTaskV2ControllerIntegrationTest` 已补 worker 驱动的 archive/extract 完成态与 extract 失败态端到端覆盖 后续未完成： -- 真实压缩任务 -- 真实解压任务 -- 任务重试策略 -- 服务重启后的 `RUNNING` 任务恢复策略 -- 更完整的任务进度字段 - 前端任务面板自动轮询或 SSE 化 - 移动端任务入口 @@ -115,11 +137,10 @@ npm run test - v2 分享后端骨架已落地 - 支持密码、过期时间、导入开关、下载次数相关字段、我的分享、删除 -- `allowDownload` 已落库并返回 +- `allowDownload` 已接入 `GET /api/v2/shares/{token}?download=1`，会校验密码、过期、下载开关和下载次数并递增 `downloadCount` 后续未完成： -- 独立 v2 下载路由消费 `allowDownload` - 前端分享二期设置界面 - 公开分享页的密码输入与过期/次数提示体验 @@ -127,14 +148,15 @@ npm run test 当前状态： -- v2 upload session 创建、查询、取消、complete、part metadata 和过期清理骨架已落地 -- 当前只记录会话和 part metadata，不做真实对象存储 multipart +- v2 upload session 后端已补齐创建、查询、取消、prepare-part、record-part、complete 和过期清理 +- `FileContentStorage` 已新增 multipart 抽象；`S3FileContentStorage` 已实现 create/upload-part/complete/abort +- 默认 S3 存储策略现在会声明 `multipartUpload=true`；创建会话时会生成 `multipartUploadId`，v2 响应会返回 `multipartUpload` +- `GET /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}/prepare` 已可返回单分片直传地址；`POST /complete` 会先提交 multipart complete，再复用旧 `FileService.completeUpload()` 落库 +- 过期清理已从普通 `deleteBlob` 升级为对未完成 multipart 执行 abort +- 前端上传队列仍未切到 v2 upload session 后续未完成： -- 在 `FileContentStorage` 抽象层补 multipart 能力语义 -- S3 multipart 的 create/upload-part/complete/abort -- 过期清理从普通 `deleteBlob` 升级为可 abort 未完成 multipart - 前端上传队列切到 v2 session ### P2：存储策略继续推进 @@ -149,8 +171,7 @@ npm run test - 管理台新增/编辑/停用策略 - 多策略迁移任务 -- 按策略能力决定上传路径 -- 真正启用 multipart 能力 +- 按策略能力决定上传路径与前端上传策略 ## 当前本地运行状态 diff --git a/docs/api-reference.md b/docs/api-reference.md index bc540a9..37a6cb6 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -177,7 +177,7 @@ 控制器： -- `backend/src/main/java/com/yoyuzh/files/FileController.java` +- `backend/src/main/java/com/yoyuzh/files/core/FileController.java` ### 3.1 上传相关 @@ -246,6 +246,25 @@ - 登录用户可将分享内容导入自己的网盘 - 普通文件导入时会新建自己的 `StoredFile` 并复用源 `FileBlob`，不会再次写入物理文件 +### 3.6 v2 上传会话 + +- `POST /api/v2/files/upload-sessions` +- `GET /api/v2/files/upload-sessions/{sessionId}` +- `DELETE /api/v2/files/upload-sessions/{sessionId}` +- `GET /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}/prepare` +- `PUT /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}` +- `POST /api/v2/files/upload-sessions/{sessionId}/complete` + +说明： + +- 需要登录，只允许操作当前用户自己的上传会话 +- 会话响应返回 `sessionId`、`objectKey`、`multipartUpload`、`path`、`filename`、`contentType`、`size`、`storagePolicyId`、`status`、`chunkSize`、`chunkCount`、`expiresAt`、`createdAt`、`updatedAt` +- 默认 S3 存储策略下，创建会话时会立即初始化 multipart upload，并把 `multipartUpload=true` 返回给客户端；本地策略仍会返回 `multipartUpload=false` +- `GET /parts/{partIndex}/prepare` 会返回当前分片的直传信息：`direct`、`uploadUrl`、`method`、`headers`、`storageName` +- `PUT /parts/{partIndex}` 请求体仍为 `{ "etag": "...", "size": 8388608 }`，只负责记录 part 元数据，不直接接收字节流 +- `POST /complete` 会先按已记录的 part 元数据提交 multipart complete，再复用旧上传完成链路写入 `FileBlob + StoredFile + FileEntity.VERSION` +- 后端每小时清理过期且未完成的会话；若会话已绑定 multipart upload，会优先向对象存储发送 abort + ## 4. 快传模块 控制器： @@ -389,7 +408,7 @@ - 需要管理员登录 - 返回当前存储策略的只读列表和结构化能力声明 - 当前仅用于管理台查看默认策略、启用状态、存储类型和能力矩阵，不支持新增、编辑、启停或删除策略 -- `capabilities.multipartUpload` 当前仍为能力声明字段，不代表真实对象存储 multipart 已启用 +- `capabilities.multipartUpload` 现在会反映默认策略是否支持 v2 上传会话 multipart；当前默认 S3 策略为 `true`，本地策略为 `false` ## 6. 前端公开路由与接口关系 @@ -452,11 +471,11 @@ - 本阶段不新增对外 API，`/api/files/**`、分享、回收站、快传导入等响应结构保持不变。 - 后端在旧接口内部开始双写实体模型：上传完成、外部导入、分享导入和网盘复制会继续写 `FileBlob`，同时创建或复用 `FileEntity.VERSION`，并写入 `StoredFile.primaryEntity` 与 `StoredFileEntity(PRIMARY)`。 - 下载、分享详情、回收站、ZIP 下载仍读取 `StoredFile.blob`；后续阶段稳定后再切换到 `primaryEntity` 读取。 -- 2026-04-08 阶段 3 第一小步 API 补充：新增受保护的 v2 上传会话骨架接口，`POST /api/v2/files/upload-sessions` 创建会话，`GET /api/v2/files/upload-sessions/{sessionId}` 查询当前用户自己的会话，`DELETE /api/v2/files/upload-sessions/{sessionId}` 取消会话。当前响应只返回 `sessionId`、`objectKey`、路径、文件名、状态、分片大小、分片数量和时间字段；实际文件内容仍走旧上传链路，尚未开放 v2 分片上传/完成接口。 -- 2026-04-08 阶段 3 第二小步 API 补充：新增 `POST /api/v2/files/upload-sessions/{sessionId}/complete`，用于把当前用户自己的上传会话提交完成。该接口当前不接收请求体，会复用会话里的 `objectKey/path/filename/contentType/size` 调用旧上传完成落库链路，成功后返回 `COMPLETED` 状态的 v2 会话响应；分片内容上传端点仍未开放。 -- 2026-04-08 阶段 3 第三小步 API 补充：新增 `PUT /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}`，请求体为 `{ "etag": "...", "size": 8388608 }`，用于记录当前用户上传会话的 part 元数据并返回 v2 会话响应。该接口会校验 part 范围和会话状态，当前只更新 `uploadedPartsJson`，不接收或合并真实文件分片内容。 -- 2026-04-08 阶段 3 第四小步 API 补充：本小步没有新增对外 API。后端新增上传会话过期清理任务，只处理未完成且已过期的会话，并把它们标记为 `EXPIRED`；已完成会话和旧 `/api/files/**` 上传接口响应不变。 -- 2026-04-08 阶段 4 第一小步 API 补充：本小步没有新增存储策略管理 API。v2 上传会话响应新增 `storagePolicyId`，用于标识该会话绑定的默认存储策略；当前该字段只服务后续 multipart/多策略迁移，旧 `/api/files/**` 上传下载接口响应不变。 +- 2026-04-08 阶段 3 第一小步 API 补充：新增受保护的 v2 上传会话接口族，`POST /api/v2/files/upload-sessions` 创建会话，`GET /api/v2/files/upload-sessions/{sessionId}` 查询当前用户自己的会话，`DELETE /api/v2/files/upload-sessions/{sessionId}` 取消会话。当前响应会返回 `sessionId`、`objectKey`、`multipartUpload`、路径、文件名、状态、分片大小、分片数量和时间字段。 +- 2026-04-08 阶段 3 第二小步 API 补充：`POST /api/v2/files/upload-sessions/{sessionId}/complete` 用于把当前用户自己的上传会话提交完成。当前默认 S3 策略下，该接口会先完成 multipart 合并，再复用旧上传完成链路落库，成功后返回 `COMPLETED` 状态的 v2 会话响应。 +- 2026-04-08 阶段 3 第三小步 API 补充：`PUT /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}` 请求体为 `{ "etag": "...", "size": 8388608 }`，用于记录当前用户上传会话的 part 元数据并返回 v2 会话响应；`GET /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}/prepare` 则返回该分片的直传地址和请求头。字节流仍直接上传到对象存储，不经过后端转发。 +- 2026-04-08 阶段 3 第四小步 API 补充：本小步没有新增额外资源类型。后端新增上传会话过期清理任务，只处理未完成且已过期的会话，并把它们标记为 `EXPIRED`；若会话绑定了 multipart upload，还会在清理时发起 abort。 +- 2026-04-08 阶段 4 第一小步 API 补充：本小步没有新增存储策略管理 API。v2 上传会话响应新增 `storagePolicyId`，用于标识该会话绑定的默认存储策略；该字段现在也用于区分会话是否应按策略能力走 multipart 上传。 ## 2026-04-08 阶段 5 文件搜索第一小步 @@ -546,13 +565,26 @@ 需要登录。取消当前用户自己的任务，`QUEUED` / `RUNNING` 会转为 `CANCELLED` 并写入 `finishedAt`，终态任务保持原样。 +`POST /api/v2/tasks/{id}/retry` + +需要登录。仅允许当前用户重试自己处于 `FAILED` 的后台任务。 + +补充说明： + +- 成功后任务状态会重置为 `QUEUED` +- `finishedAt` 与 `errorMessage` 会被清空 +- `publicStateJson.phase` 会重置为 `queued` +- `publicStateJson.attemptCount` 会重置为 `0` +- 公开 state 会按服务端保存的 `privateStateJson` 重建，因此失败执行时写入的瞬时字段不会保留 +- 非 `FAILED` 任务调用会返回 `400` + `POST /api/v2/tasks/archive` -需要登录。创建 `ARCHIVE` 类型的 `QUEUED` 任务；`fileId` 必须属于当前用户且未删除，`path` 必须匹配服务端派生逻辑路径，暂允许文件和目录；当前 worker 只做 no-op 占位完成，不执行真实压缩。 +需要登录。创建 `ARCHIVE` 类型的 `QUEUED` 任务；`fileId` 必须属于当前用户且未删除，`path` 必须匹配服务端派生逻辑路径，暂允许文件和目录；当前 worker 会生成 zip 并把归档结果回写到原文件同级目录。 `POST /api/v2/tasks/extract` -需要登录。创建 `EXTRACT` 类型的 `QUEUED` 任务；`fileId` 必须属于当前用户且未删除，`path` 必须匹配服务端派生逻辑路径，并拒绝目录和非压缩包类文件；当前 worker 只做 no-op 占位完成，不执行真实解压。 +需要登录。创建 `EXTRACT` 类型的 `QUEUED` 任务；`fileId` 必须属于当前用户且未删除，`path` 必须匹配服务端派生逻辑路径，并拒绝目录和非压缩包类文件；当前 worker 只支持 zip-compatible 归档，会剥离共享根目录，并把解压结果恢复到原文件父目录。 `POST /api/v2/tasks/media-metadata` @@ -561,6 +593,12 @@ 补充说明： - worker 会定时领取少量 `QUEUED` 任务并切换为 `RUNNING`，完成后标记 `COMPLETED`，异常时标记 `FAILED` 并写入 `errorMessage`。 +- `publicStateJson.phase` 当前会经历 `queued -> running -> archiving/extracting/extracting-metadata -> completed/failed/cancelled` 这样的最小阶段流转。 +- `publicStateJson` 还会暴露 `attemptCount/maxAttempts`；当前默认预算为 `ARCHIVE=4`、`EXTRACT=3`、`MEDIA_META=2`。 +- 任务进入 `RUNNING` 后，`publicStateJson` 会额外暴露 `workerOwner/heartbeatAt/leaseExpiresAt/startedAt`，用于描述当前 worker 的 lease 和 heartbeat；终态或重排回队列后会移除运行态 owner/lease 字段。 +- `ARCHIVE/EXTRACT` 任务还会在 `publicStateJson` 里暴露 `processedFileCount/totalFileCount`、`processedDirectoryCount/totalDirectoryCount` 与真实 `progressPercent`；`MEDIA_META` 会额外暴露 `metadataStage`。 +- 当 worker 命中失败时，任务会按失败分类写入 `failureCategory`。`TRANSIENT_INFRASTRUCTURE`、`RATE_LIMITED` 与部分 `UNKNOWN` 失败会按任务类型退避自动重排回 `QUEUED`，并在 `publicStateJson` 写入 `retryScheduled=true`、`nextRetryAt`、`retryDelaySeconds`、`lastFailureMessage`、`lastFailureAt`；`UNSUPPORTED_INPUT` 与 `DATA_STATE` 这类确定性失败不会自动重试。 - 已取消或其他终态任务不会被重新执行。 +- 服务重启后，只有 lease 已过期或历史上没有 lease 的 `RUNNING` 任务会在启动完成时被重置回 `QUEUED`，避免多实例下误抢仍在运行的 worker。 - 创建成功后的任务 state 使用服务端文件信息，至少包含 `fileId`、`path`、`filename`、`directory`、`contentType`、`size`。 - 桌面端 `Files` 页面会拉取最近 10 条任务、提供 `QUEUED/RUNNING` 取消按钮，并可为当前选中文件创建 `MEDIA_META` 任务；移动端与 archive/extract 的前端入口暂未接入。 diff --git a/docs/architecture.md b/docs/architecture.md index 644c0bf..3e952da 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -74,7 +74,14 @@ 后端包结构： - `com.yoyuzh.auth` -- `com.yoyuzh.files` +- `com.yoyuzh.files.core` +- `com.yoyuzh.files.upload` +- `com.yoyuzh.files.share` +- `com.yoyuzh.files.search` +- `com.yoyuzh.files.events` +- `com.yoyuzh.files.tasks` +- `com.yoyuzh.files.storage` +- `com.yoyuzh.files.policy` - `com.yoyuzh.transfer` - `com.yoyuzh.admin` - `com.yoyuzh.config` @@ -124,8 +131,15 @@ 核心文件： -- `backend/src/main/java/com/yoyuzh/files/FileController.java` -- `backend/src/main/java/com/yoyuzh/files/FileService.java` +- `backend/src/main/java/com/yoyuzh/files/core/FileController.java` +- `backend/src/main/java/com/yoyuzh/files/core/FileService.java` +- `backend/src/main/java/com/yoyuzh/files/upload/UploadSessionService.java` +- `backend/src/main/java/com/yoyuzh/files/share/ShareV2Service.java` +- `backend/src/main/java/com/yoyuzh/files/search/FileSearchService.java` +- `backend/src/main/java/com/yoyuzh/files/events/FileEventService.java` +- `backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskService.java` +- `backend/src/main/java/com/yoyuzh/files/policy/StoragePolicyService.java` +- `backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Controller.java` - `backend/src/main/java/com/yoyuzh/files/storage/*` - `front/src/pages/Files.tsx` @@ -140,6 +154,7 @@ 关键实现说明： +- `com.yoyuzh.files` 已按职责拆成 `core/upload/share/search/events/tasks/storage/policy` 八个子包，控制器路径、数据库表结构、接口路径和前端调用方式保持不变；这次调整只做包重组与引用修正，不改业务语义 - 文件元数据在数据库 - 文件内容通过独立 `FileBlob` 实体映射到底层对象；`StoredFile` 只负责用户、目录、文件名、路径、分享关系等逻辑元数据 - 新文件的物理对象 key 使用全局 `blobs/...` 命名，不再把 `userId/path` 编进对象 key @@ -151,6 +166,7 @@ - 定时清理任务会删除超过 10 天的回收站条目；只有当某个 `FileBlob` 的最后一个逻辑引用随之消失时，才真正删除底层对象 - 应用启动时会把旧 `portal_file.storage_name` 行自动回填到新的 `blob_id` 引用，保证存量数据能继续读取 - 当前线上网盘文件存储已切到多吉云对象存储，后端先通过多吉云临时密钥 API 换取短期 S3 会话，再访问底层 COS 兼容桶 +- v2 上传会话后端现已支持按存储策略能力走真实 multipart：默认 S3 策略会在创建会话时初始化 `multipartUploadId`，分片上传通过预签名 `UploadPart` 直传对象存储，完成时先提交 multipart complete，再复用旧 `FileService.completeUpload()` 落库；本地策略仍保持 `multipartUpload=false` - 前端会缓存目录列表和最后访问路径 - 桌面网盘页在左侧树状目录栏底部固定展示回收站入口；移动端在网盘页顶部提供回收站入口；两端共用独立 `RecycleBin` 页面调用 `/api/files/recycle-bin` 与恢复接口 @@ -446,13 +462,14 @@ Android 壳补充说明： - `StoredFile.blob` 仍是当前生产读取路径；`StoredFile.primaryEntity` 与关系表暂时只作为兼容迁移数据，不影响旧 `/api/files/**` DTO 和前端调用。 - `portal_stored_file_entity.stored_file_id` 随 `portal_file` 删除级联清理；`portal_file_entity.created_by` 在用户删除时置空，避免实体审计关系阻塞用户清理。 - 2026-04-08 阶段 3 第一小步补充：后端新增上传会话二期最小骨架。`UploadSession` 记录用户、目标路径、文件名、对象键、分片大小、分片数量、状态、过期时间和已上传分片占位 JSON；`/api/v2/files/upload-sessions` 目前只提供创建、查询、取消会话，不承接实际分片内容上传，也不替换旧 `/api/files/upload/**` 生产链路。 -- 2026-04-08 阶段 3 第二小步补充：上传会话新增完成状态机。`UploadSessionService.completeOwnedSession()` 会复用旧 `FileService.completeUpload()` 完成对象确认、目录补齐、配额/冲突校验和 `FileBlob + StoredFile + FileEntity.VERSION` 双写落库，然后把会话标记为 `COMPLETED`；失败时标记 `FAILED`，过期时标记 `EXPIRED`。当前仍没有独立 v2 分片内容写入端点。 +- 2026-04-08 阶段 3 第二小步补充：上传会话新增完成状态机。`UploadSessionService.completeOwnedSession()` 会复用旧 `FileService.completeUpload()` 完成对象确认、目录补齐、配额/冲突校验和 `FileBlob + StoredFile + FileEntity.VERSION` 双写落库，然后把会话标记为 `COMPLETED`；失败时标记 `FAILED`，过期时标记 `EXPIRED`。当时仍没有独立 v2 分片内容写入端点。 - 2026-04-08 阶段 3 第三小步补充：上传会话新增 part 状态记录。`UploadSessionService.recordUploadedPart()` 会校验会话归属、状态、过期时间和 part 范围，把 `etag/size/uploadedAt` 写入 `uploadedPartsJson`，并将新会话推进到 `UPLOADING`。当前实现是会话状态跟踪，不是跨存储驱动的分片内容写入/合并实现。 - 2026-04-08 阶段 3 第四小步补充：上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话，尝试删除 `objectKey` 对应的临时 blob，然后标记为 `EXPIRED`。已完成文件不参与清理，避免误删已经落库的生产对象。 -- 2026-04-08 阶段 4 第一小步补充：后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略，并把当前 `FileStorageProperties` 映射为 `LOCAL` 或 `S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON；当前能力声明中 `multipartUpload=false`，用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID，但 `FileContentStorage` 仍保持单对象上传/校验抽象，旧 `/api/files/**` 生产路径不切换。 -- 2026-04-08 `files/storage` 合并补充：S3 存储实现拆出多吉云临时密钥客户端与运行期会话提供器。`S3FileContentStorage` 现在通过 `S3SessionProvider.currentSession()` 获取当前 bucket、`S3Client` 和 `S3Presigner`，避免每次操作重复内联多吉云 token 解析逻辑；测试环境可直接注入 mock S3 client/presigner。该改动没有引入 multipart，仍是单对象 PUT/HEAD/GET/COPY/DELETE 路径。 +- 2026-04-08 阶段 4 第一小步补充：后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略，并把当前 `FileStorageProperties` 映射为 `LOCAL` 或 `S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON；当时能力声明里的 `multipartUpload=false` 用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID，但旧 `/api/files/**` 生产路径当时仍不切换。 +- 2026-04-08 `files/storage` 合并补充：S3 存储实现拆出多吉云临时密钥客户端与运行期会话提供器。`S3FileContentStorage` 现在通过 `S3SessionProvider.currentSession()` 获取当前 bucket、`S3Client` 和 `S3Presigner`，避免每次操作重复内联多吉云 token 解析逻辑；测试环境可直接注入 mock S3 client/presigner。当时该改动还没有引入 multipart，仍是单对象 PUT/HEAD/GET/COPY/DELETE 路径。 - 2026-04-08 阶段 4 第二小步补充：`FileService` 在创建新的 `FileEntity.VERSION` 时会通过 `StoragePolicyService.ensureDefaultPolicy()` 写入默认 `storagePolicyId`；`FileEntityBackfillService` 对历史 `FileBlob` 回填新实体时也写入同一默认策略。复用已有实体时保持原策略字段不变，只增加引用计数，避免在兼容迁移阶段覆盖历史数据。 -- 2026-04-08 阶段 4 第三小步补充：管理台新增只读存储策略列表。`AdminController` 暴露 `GET /api/admin/storage-policies`，`AdminService` 通过白名单 DTO 返回策略基础字段和结构化 `StoragePolicyCapabilities`；前端 `react-admin` 新增 `storagePolicies` 资源展示能力矩阵。该能力只做配置可视化，不改变旧上传下载路径，不暴露凭证，不启用策略编辑或 multipart。 +- 2026-04-08 阶段 4 第三小步补充：管理台新增只读存储策略列表。`AdminController` 暴露 `GET /api/admin/storage-policies`，`AdminService` 通过白名单 DTO 返回策略基础字段和结构化 `StoragePolicyCapabilities`；前端 `react-admin` 新增 `storagePolicies` 资源展示能力矩阵。该能力只做配置可视化，不改变旧上传下载路径，也不暴露凭证或提供策略编辑能力。 +- 2026-04-09 上传会话二期补充：`FileContentStorage` 抽象已新增 `createMultipartUpload/prepareMultipartPartUpload/completeMultipartUpload/abortMultipartUpload`；`S3FileContentStorage` 基于预签名 `UploadPart` 与 S3 `Complete/AbortMultipartUpload` 实现真实 multipart。`UploadSession` 新增 `multipartUploadId`，`UploadSessionService.createSession()` 会在默认策略声明 `multipartUpload=true` 时初始化 uploadId，并通过 `GET /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}/prepare` 返回单分片直传地址。会话完成时先按 `uploadedPartsJson` 提交 multipart complete，再复用旧上传完成链路落库；过期清理则改为优先 abort 未完成 multipart。 ## 2026-04-08 阶段 5 文件搜索第一小步 @@ -469,8 +486,8 @@ Android 壳补充说明： - 旧分享仍保留在 `/api/files/share-links/**`，用于兼容当前前端公开分享页和旧导入路径。 - 新 v2 分享位于 `com.yoyuzh.api.v2.shares` 与 `ShareV2Service`；`FileShareLink` 新增 `passwordHash`、`expiresAt`、`maxDownloads`、`downloadCount`、`viewCount`、`allowImport`、`allowDownload`、`shareName` 策略字段。 -- 公开端点仅包括 `GET /api/v2/shares/{token}` 与 `POST /api/v2/shares/{token}/verify-password`；创建、导入、我的分享列表和删除仍需要登录。 -- 密码分享在校验前隐藏 `file` 详情；v2 导入会在复用旧导入落库链路前校验过期时间、密码、`allowImport` 和 `maxDownloads`。当前 `allowDownload` 只落库和返回，尚未接入独立 v2 下载路由。 +- 公开端点包括 `GET /api/v2/shares/{token}`、`POST /api/v2/shares/{token}/verify-password`，以及 `GET /api/v2/shares/{token}?download=1`；创建、导入、我的分享列表和删除仍需要登录。 +- 密码分享在校验前隐藏 `file` 详情；v2 导入会在复用旧导入落库链路前校验过期时间、密码、`allowImport` 和 `maxDownloads`。v2 下载也会统一校验过期时间、密码、`allowDownload` 和 `maxDownloads`，成功后复用现有文件下载链路并递增 `downloadCount`。 ## 2026-04-08 阶段 5 文件事件流最小闭环 @@ -482,8 +499,14 @@ Android 壳补充说明： ## 2026-04-08 阶段 6 任务框架与 worker 后端最小骨架 - 后端新增 `BackgroundTask` / `BackgroundTaskType` / `BackgroundTaskStatus` / `BackgroundTaskRepository` / `BackgroundTaskService`，用于承载后续压缩、解压、缩略图、媒体元数据和清理类后台工作。 -- 新增受保护的 `/api/v2/tasks/**`：`GET /api/v2/tasks`、`GET /api/v2/tasks/{id}`、`DELETE /api/v2/tasks/{id}`，以及 `POST /api/v2/tasks/archive`、`POST /api/v2/tasks/extract`、`POST /api/v2/tasks/media-metadata` 占位创建接口。 -- 任务创建入口集中在 `BackgroundTaskService` 校验 `StoredFile`：`fileId` 必须属于当前用户且未删除，请求 `path` 必须匹配由 `StoredFile.path + filename` 派生的真实逻辑路径；`ARCHIVE` 暂允许文件和目录，`EXTRACT` 仅允许压缩包类文件，`MEDIA_META` 仅允许媒体类文件。任务 public/private state 使用服务端派生的 `fileId`、`path`、`filename`、`directory`、`contentType`、`size`。 -- 当前实现新增了 worker 最小调度：定时扫描少量 `QUEUED` 任务，通过状态条件更新完成 claim，`MEDIA_META` 任务会进入独立 handler 写入基础媒体元数据与图片宽高，其余任务类型执行 no-op handler 后标记 `COMPLETED`；handler 异常会标记 `FAILED` 并记录错误原因，已取消任务不会被领取。 -- 当前仍不包含真实压缩、解压、缩略图、媒体元数据解析、重试/恢复策略或前端队列展示。 +- 新增受保护的 `/api/v2/tasks/**`：`GET /api/v2/tasks`、`GET /api/v2/tasks/{id}`、`DELETE /api/v2/tasks/{id}`、`POST /api/v2/tasks/{id}/retry`，以及 `POST /api/v2/tasks/archive`、`POST /api/v2/tasks/extract`、`POST /api/v2/tasks/media-metadata` 创建接口。 +- 任务创建入口集中在 `BackgroundTaskService` 校验 `StoredFile`：`fileId` 必须属于当前用户且未删除，请求 `path` 必须匹配由 `StoredFile.path + filename` 派生的真实逻辑路径；`ARCHIVE` 允许文件和目录，`EXTRACT` 当前只允许 zip-compatible 文件（`.zip/.jar/.war` 或 zip/java archive 内容类型），`MEDIA_META` 仅允许媒体类文件。任务 public/private state 使用服务端派生的 `fileId`、`path`、`filename`、`directory`、`contentType`、`size`；其中 `ARCHIVE` 还会写入 `outputPath/outputFilename`，`EXTRACT` 会写入 `outputPath/outputDirectoryName`。 +- 当前实现新增了 worker 调度与多实例 lease：定时先回收 lease 已过期的 `RUNNING` 任务，再扫描少量 `QUEUED` 任务，通过状态条件更新完成 claim，并写入持久化 `leaseOwner/leaseExpiresAt/heartbeatAt` 与公开 `workerOwner/heartbeatAt/leaseExpiresAt/startedAt`。运行中所有 progress/完成/失败更新都要求 owner 匹配，丢失 lease 的旧 worker 不会覆盖新状态。 +- `MEDIA_META` 任务会进入独立 handler 写入基础媒体元数据与图片宽高，并在公开 state 写入 `metadataStage`；`ARCHIVE` 任务会调用 `FileService.buildArchiveBytes(...)` 生成 zip 并回写同级目录；`EXTRACT` 任务会读取 zip-compatible 归档、剥离共享根目录或把单文件直接恢复到父目录，再通过 `FileService.importExternalFilesAtomically(...)` 做预检、批量导入和失败 blob 清理。 +- `BackgroundTaskService` 还会在 `publicStateJson` 里统一维护最小进度阶段 `phase`：创建时是 `queued`，claim 后进入 `running`，worker 开始执行时按任务类型细化成 `archiving` / `extracting` / `extracting-metadata`，完成/失败/取消时再收口为 `completed` / `failed` / `cancelled`。 +- `ARCHIVE` 与 `EXTRACT` 任务现在会在运行和完成阶段暴露真实条目计数：`processedFileCount/totalFileCount`、`processedDirectoryCount/totalDirectoryCount`，并基于真实总量计算 `progressPercent`。其中 `ARCHIVE` 按实际写入 zip entry 推进，`EXTRACT` 按实际创建目录和导入文件推进；`MEDIA_META` 则暴露阶段型 `metadataStage`。 +- 当前 `POST /api/v2/tasks/{id}/retry` 已支持最小手动重试：只有 `FAILED` 任务可以被当前用户重置回 `QUEUED`，并清空 `finishedAt/errorMessage`，按 `privateStateJson` 重建公开 state，同时把 `attemptCount` 重置回 0。 +- `BackgroundTaskStartupRecovery` 现在只会在服务启动完成后回收 lease 已过期或历史上缺少 lease 的 `RUNNING` 任务，恢复时按 `privateStateJson` 重建公开 state；不会再无条件重排所有 `RUNNING` 任务。 +- worker 现在会按失败分类和任务类型做自动重试：失败会归到 `UNSUPPORTED_INPUT`、`DATA_STATE`、`TRANSIENT_INFRASTRUCTURE`、`RATE_LIMITED`、`UNKNOWN`；其中 `ARCHIVE` 默认最多 4 次、`EXTRACT` 最多 3 次、`MEDIA_META` 最多 2 次，公开 state 会暴露 `attemptCount/maxAttempts/retryScheduled/nextRetryAt/retryDelaySeconds/lastFailureMessage/lastFailureAt/failureCategory`。 +- 当前仍不包含非 zip 解压格式、缩略图/视频时长任务，以及 archive/extract 的前端入口。 - 桌面端 `front/src/pages/Files.tsx` 已接入最近 10 条后台任务查看与取消入口，并可为当前选中文件创建 `MEDIA_META` 任务；移动端与 archive/extract 的前端入口仍未接入。 diff --git a/memory.md b/memory.md index d21f72f..245c07c 100644 --- a/memory.md +++ b/memory.md @@ -139,8 +139,8 @@ - 管理员改密接口: `backend/src/main/java/com/yoyuzh/admin/AdminService.java` - 管理台统计与 7 天上线记录: `backend/src/main/java/com/yoyuzh/admin/AdminMetricsService.java`、`backend/src/main/java/com/yoyuzh/admin/AdminDailyActiveUserEntity.java`、`backend/src/main/java/com/yoyuzh/config/JwtAuthenticationFilter.java` - 管理台 dashboard 展示与请求折线图: `front/src/admin/dashboard.tsx`、`front/src/admin/dashboard-state.ts` - - 网盘 blob 模型与回填: `backend/src/main/java/com/yoyuzh/files/FileService.java`、`backend/src/main/java/com/yoyuzh/files/FileBlob.java`、`backend/src/main/java/com/yoyuzh/files/FileBlobBackfillService.java` - - 网盘回收站与恢复: `backend/src/main/java/com/yoyuzh/files/FileService.java`、`backend/src/main/java/com/yoyuzh/files/FileController.java`、`backend/src/main/java/com/yoyuzh/files/StoredFile.java`、`front/src/pages/RecycleBin.tsx`、`front/src/pages/recycle-bin-state.ts` + - 网盘 blob 模型与回填: `backend/src/main/java/com/yoyuzh/files/core/FileService.java`、`backend/src/main/java/com/yoyuzh/files/core/FileBlob.java`、`backend/src/main/java/com/yoyuzh/files/core/FileBlobBackfillService.java` + - 网盘回收站与恢复: `backend/src/main/java/com/yoyuzh/files/core/FileService.java`、`backend/src/main/java/com/yoyuzh/files/core/FileController.java`、`backend/src/main/java/com/yoyuzh/files/core/StoredFile.java`、`front/src/pages/RecycleBin.tsx`、`front/src/pages/recycle-bin-state.ts` - 前端生产 API 基址: `front/.env.production` - Capacitor Android 入口与配置: `front/capacitor.config.ts`、`front/android/` ## 2026-04-08 阶段 1 升级记录 @@ -153,7 +153,7 @@ - 已新增文件实体模型二期的兼容表模型：`FileEntity`、`StoredFileEntity`、`FileEntityType`，并在 `StoredFile` 上新增 `primaryEntity` 与 `updatedAt`。 - 已新增 `FileEntityBackfillService`，启动后在旧 `FileBlob` 仍保留的前提下，把已有 `StoredFile.blob` 只增量映射到 `FileEntity.VERSION` 与 `StoredFile.primaryEntity`；现有下载、复制、移动、分享、回收站读写路径暂不切换。 -- 当前阶段未删除 `FileBlob`，未切换前端，未引入上传会话二期。 +- 当时阶段未删除 `FileBlob`，未切换前端，也还未引入上传会话二期。 ## 2026-04-08 阶段 2 第二小步记录 - 文件写入路径开始双写 `FileBlob + FileEntity.VERSION`：普通代理上传、直传完成、外部文件导入、分享导入，以及网盘复制复用 blob 时，都会给新 `StoredFile` 写入 `primaryEntity` 并创建 `StoredFileEntity(PRIMARY)` 关系。 @@ -164,13 +164,22 @@ - 2026-04-08 阶段 3 第三小步：新增 `PUT /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}`，用于记录当前用户上传会话的 part 元数据到 `uploadedPartsJson`，并把会话状态从 `CREATED` 推进到 `UPLOADING`；该接口只记录 `etag/size` 等状态，不承担真正的对象存储分片内容写入或合并。 - 2026-04-08 阶段 3 第四小步：`UploadSessionService` 新增定时过期清理，按小时扫描 `CREATED/UPLOADING/COMPLETING` 且已过期的会话，尝试删除对应临时 `blobs/...` 对象，并把会话标记为 `EXPIRED`；`COMPLETED/CANCELLED/FAILED/EXPIRED` 不在本轮清理范围内。 - 2026-04-08 multipart 评估结论：暂不把 v2 上传会话直接接入真实对象存储分片写入/合并。当前 `FileContentStorage` 仍是单对象上传/校验抽象，缺少 multipart uploadId、part URL 预签名、complete/abort 语义；立即接入会把上传会话写死在当前多吉云 S3 配置上，并让过期清理误以为 `deleteBlob` 能释放未完成分片。下一步先做阶段 4 存储策略与能力声明骨架，再按 `multipartUpload` 能力接 S3 multipart。 -- 2026-04-08 阶段 4 第一小步：新增 `StoragePolicy`、`StoragePolicyType`、`StoragePolicyCredentialMode`、`StoragePolicyCapabilities` 与 `StoragePolicyService`，启动时把当前 `app.storage.provider` 映射成一条默认策略；本地策略声明 `serverProxyDownload=true`、`multipartUpload=false`，多吉云/S3 兼容策略声明 `directUpload=true`、`signedDownloadUrl=true`、`requiresCors=true`、`multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`，但旧上传下载路径和前端上传队列仍未切换。 +- 2026-04-08 阶段 4 第一小步：新增 `StoragePolicy`、`StoragePolicyType`、`StoragePolicyCredentialMode`、`StoragePolicyCapabilities` 与 `StoragePolicyService`，启动时把当前 `app.storage.provider` 映射成一条默认策略；当时本地策略声明 `serverProxyDownload=true`、`multipartUpload=false`，多吉云/S3 兼容策略也先声明为 `directUpload=true`、`signedDownloadUrl=true`、`requiresCors=true`、`multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`，但旧上传下载路径和前端上传队列仍未切换。 - 2026-04-08 合并 `files/storage` 补提交后修复：`S3FileContentStorage` 改为复用 `DogeCloudS3SessionProvider` / `DogeCloudTmpTokenClient` 获取并缓存运行期 `S3Client` 与 `S3Presigner`，保留生产构造器 `S3FileContentStorage(FileStorageProperties)`，同时提供测试用注入构造器；S3 直传、签名下载、上传校验、读旧对象键 fallback、rename/move/copy、离线快传对象读写继续通过 `FileContentStorage` 统一抽象。 - 2026-04-08 阶段 4 第二小步：新写入和回填生成的 `FileEntity.VERSION` 会记录默认 `StoragePolicy.id` 到 `storagePolicyId`，让物理实体可以追踪归属存储策略；复用已有 `FileEntity` 时只增加引用计数，不覆盖历史实体策略字段。旧 `/api/files/**` 读取路径仍继续依赖 `StoredFile.blob`。 -- 2026-04-08 阶段 4 第三小步：新增管理员只读存储策略查看能力，后端暴露 `GET /api/admin/storage-policies`，前端管理台新增“存储策略”资源列表和能力矩阵展示；该接口只返回白名单 DTO 与结构化 `StoragePolicyCapabilities`，不暴露凭证、不支持新增/编辑/启停/删除策略，也不启用真实 multipart。 +- 2026-04-08 阶段 4 第三小步：新增管理员只读存储策略查看能力，后端暴露 `GET /api/admin/storage-policies`，前端管理台新增“存储策略”资源列表和能力矩阵展示；该接口只返回白名单 DTO 与结构化 `StoragePolicyCapabilities`，不暴露凭证，也不支持新增/编辑/启停/删除策略。 - 2026-04-08 阶段 5 第一小步：新增用户侧 v2 文件搜索最小闭环，后端暴露受保护的 `GET /api/v2/files/search`，复用 `StoredFile` 查询当前用户未删除文件，支持 `name`、`type=file|directory|folder|all`、`sizeGte/sizeLte`、`createdGte/createdLte`、`updatedGte/updatedLte` 与分页；同时新增 `FileMetadata` / `FileMetadataRepository` 扩展表骨架，暂不迁移回收站字段、暂不接入标签/metadata 过滤、暂不改前端上传队列和旧 `/api/files/**` 行为。 - 2026-04-08 阶段 5 第二小步：前端桌面端接入最小搜索下游，新增 `front/src/lib/file-search.ts` 和 `front/src/lib/file-search.test.ts`，桌面 `front/src/pages/Files.tsx` 可通过 v2 search 单独搜索并展示结果，不写入 `getFilesListCacheKey(...)`，也不影响原有目录缓存和上传主链路；移动端暂未接入搜索，后续可按同一 helper 补入。 -- 2026-04-08 阶段 5 第三小步：新增分享二期后端最小骨架。`FileShareLink` 增加 `passwordHash`、`expiresAt`、`maxDownloads`、`downloadCount`、`viewCount`、`allowImport`、`allowDownload`、`shareName`；新增 `com.yoyuzh.api.v2.shares` 与 `ShareV2Service`，提供 v2 创建、公开读取、密码校验、导入、我的分享列表和删除。公开访问仅限 `GET /api/v2/shares/{token}` 与 `POST /api/v2/shares/{token}/verify-password`；创建、导入、我的分享、删除仍需登录。v2 导入会先校验过期时间、密码、`allowImport` 和 `maxDownloads`，再复用旧导入持久化链路；旧 `/api/files/share-links/**` 继续兼容。当前 `allowDownload` 只落库和返回，尚未接入独立 v2 下载路由。 +- 2026-04-08 阶段 5 第三小步：新增分享二期后端最小骨架。`FileShareLink` 增加 `passwordHash`、`expiresAt`、`maxDownloads`、`downloadCount`、`viewCount`、`allowImport`、`allowDownload`、`shareName`；新增 `com.yoyuzh.api.v2.shares` 与 `ShareV2Service`，提供 v2 创建、公开读取、密码校验、导入、我的分享列表和删除。公开访问包括 `GET /api/v2/shares/{token}`、`POST /api/v2/shares/{token}/verify-password`，以及 `GET /api/v2/shares/{token}?download=1` 下载入口；后者会统一校验过期时间、密码、`allowDownload` 和 `maxDownloads`，成功后复用现有下载链路并递增 `downloadCount`。创建、导入、我的分享、删除仍需登录；v2 导入仍会先校验过期时间、密码、`allowImport` 和 `maxDownloads`，再复用旧导入持久化链路；旧 `/api/files/share-links/**` 继续兼容。 - 2026-04-08 阶段 5 第四小步：新增文件事件流前后端最小闭环。后端落地 `FileEvent` / `FileEventType` / `FileEventRepository` / `FileEventService`，并提供受保护的 `GET /api/v2/files/events?path=/` SSE 入口；当前可按用户广播、按路径前缀过滤、按 `X-Yoyuzh-Client-Id` 抑制自身事件，首次连接会收到 `READY` 事件。前端新增 fetch-stream 版 `front/src/lib/file-events.ts`，不直接使用无法带鉴权头的原生 `EventSource`；桌面 `Files` 与移动 `MobileFiles` 已订阅当前目录事件，收到文件变更后失效当前目录缓存并刷新列表，搜索结果状态不被清空。 -- 2026-04-08 阶段 6 第一步：新增后台任务框架与 worker 最小骨架。后端新增 `BackgroundTask` / `BackgroundTaskType` / `BackgroundTaskStatus` / `BackgroundTaskRepository` / `BackgroundTaskService`，并暴露受保护的 `GET /api/v2/tasks`、`GET /api/v2/tasks/{id}`、`DELETE /api/v2/tasks/{id}` 以及 `POST /api/v2/tasks/archive`、`POST /api/v2/tasks/extract`、`POST /api/v2/tasks/media-metadata` 占位创建接口；任务创建入口已校验 `fileId` 属于当前用户、未删除、请求 `path` 匹配服务端派生逻辑路径，并按任务类型限制目录/压缩包/媒体文件，任务 state 使用服务端文件信息；当前 worker 会定时领取 `QUEUED` 任务、切换为 `RUNNING`，其中 `MEDIA_META` 已由最小真实 handler 写入基础媒体元数据与图片宽高，其余任务类型仍通过 no-op handler 标记 `COMPLETED`，异常时标记 `FAILED` 并记录错误原因。压缩/解压/缩略图/视频时长/前端任务面板仍未接入。 +- 2026-04-09 阶段 5 第五小步：上传会话二期后端接入真实 multipart。`FileContentStorage` 新增 `createMultipartUpload/prepareMultipartPartUpload/completeMultipartUpload/abortMultipartUpload` 抽象，`S3FileContentStorage` 用预签名 `UploadPart` 和 `Complete/AbortMultipartUpload` 落地实现；默认 S3 存储策略能力改为 `multipartUpload=true`。`UploadSession` 新增 `multipartUploadId`，创建会话时若默认策略支持 multipart 会立即初始化 uploadId；v2 会话响应新增 `multipartUpload`，并开放 `GET /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}/prepare` 返回单分片直传地址。完成会话时会先按已记录 part 元数据提交 multipart complete，再复用旧 `FileService.completeUpload()` 落库；过期清理也会对未完成 multipart 执行 abort。前端上传队列仍未切到这条新链路。 +- 2026-04-08 阶段 6 第一步：新增后台任务框架与 worker 最小骨架。后端新增 `BackgroundTask` / `BackgroundTaskType` / `BackgroundTaskStatus` / `BackgroundTaskRepository` / `BackgroundTaskService`，并暴露受保护的 `GET /api/v2/tasks`、`GET /api/v2/tasks/{id}`、`DELETE /api/v2/tasks/{id}` 以及 `POST /api/v2/tasks/archive`、`POST /api/v2/tasks/extract`、`POST /api/v2/tasks/media-metadata` 创建接口；任务创建入口会校验 `fileId` 属于当前用户、未删除、请求 `path` 匹配服务端派生逻辑路径，并按任务类型限制目录、zip-compatible 解压源和媒体文件，任务 state 使用服务端文件信息。 +- 2026-04-09 阶段 6 第二步：`MEDIA_META` 之外的后台任务开始真实化。`ARCHIVE` 任务现在会派生 `outputPath/outputFilename`，由 `ArchiveBackgroundTaskHandler` 复用 `FileService.buildArchiveBytes(...)` 把目录或单文件打成 zip，并通过 `importExternalFile(...)` 写回同级目录；`EXTRACT` 任务现在会派生 `outputPath/outputDirectoryName`，由 `ExtractBackgroundTaskHandler` 读取 zip-compatible 归档、剥离共享根目录、支持单文件归档直接恢复到父目录，并通过 `FileService.importExternalFilesAtomically(...)` 在预检冲突后批量落库，失败时清理已写入的 `blobs/...`，避免留下孤儿 blob。worker 仍按 `QUEUED -> RUNNING -> COMPLETED/FAILED` 驱动，当前未实现非 zip 解压格式、缩略图/视频时长，以及 archive/extract 的前端入口。 +- 2026-04-09 阶段 6 第三步：后台任务新增最小 progress 字段，但仍不做假百分比。`BackgroundTaskService` 现在会在 `publicStateJson` 里统一维护 `phase`：创建时为 `queued`，claim 后为 `running`，worker 开始执行时按任务类型细化成 `archiving` / `extracting` / `extracting-metadata`，完成/失败/取消时分别收口为 `completed` / `failed` / `cancelled`。`GET /api/v2/tasks/**` 会直接透出这些阶段；`BackgroundTaskV2ControllerIntegrationTest` 也已覆盖 archive/extract 完成态、extract 失败态和取消态的 phase 回读。 +- 2026-04-09 阶段 6 第六步：`ARCHIVE/EXTRACT` 后台任务补了真实条目计数进度。worker 现在会把 progress reporter 传入 handler；`ARCHIVE` 会按实际写入 zip entry 推进 `processedFileCount/totalFileCount` 与 `processedDirectoryCount/totalDirectoryCount`，`EXTRACT` 会按实际创建目录和导入文件推进同一组字段。重试和启动恢复仍按 `privateStateJson` 重建公开 state，因此这些运行期计数字段不会被错误保留到下一次执行。 +- 2026-04-09 阶段 6 第四步：后台任务补了最小手动重试闭环。后端新增 `POST /api/v2/tasks/{id}/retry`，只允许当前用户把自己 `FAILED` 状态的任务重新置回 `QUEUED`；重试时会清空 `finishedAt/errorMessage`，按 `privateStateJson` 重建公开 state，并把 `publicStateJson.phase` 重置为 `queued`，不会保留失败时写入的 `worker` 等瞬时字段。 +- 2026-04-09 阶段 6 第五步：后台任务补了服务启动时的 `RUNNING` 恢复。最初版本会在 `ApplicationReadyEvent` 后直接把遗留 `RUNNING` 任务重排回 `QUEUED`；2026-04-09 晚些时候又升级为只回收 lease 已过期或旧数据里缺少 lease 的 `RUNNING` 任务，避免多实例场景误抢活跃 worker。 +- 2026-04-09 阶段 6 第七步：后台任务补了保守的自动重试/退避骨架。`BackgroundTask` 现在有 `attemptCount/maxAttempts/nextRunAt`；最初 `ARCHIVE`、`EXTRACT`、`MEDIA_META` 都默认最多执行 3 次，worker claim 时会递增 `attemptCount`。同日后续又升级为按任务类型区分预算与退避：`ARCHIVE` 最多 4 次、`EXTRACT` 最多 3 次、`MEDIA_META` 最多 2 次；失败分类从布尔可重试升级为 `UNSUPPORTED_INPUT/DATA_STATE/TRANSIENT_INFRASTRUCTURE/RATE_LIMITED/UNKNOWN`，公开 state 会写入 `failureCategory` 与 `retryDelaySeconds`，并按类别和任务类型决定是否自动回队列及退避时长。 +- 2026-04-09 阶段 6 第八步：后台任务补了运行期 heartbeat 与多实例 lease。`BackgroundTask` 现在持久化 `leaseOwner/leaseExpiresAt/heartbeatAt`；worker 每次 claim 会写入唯一 `workerOwner` 并续租，运行中 progress/完成/失败都会刷新 heartbeat。`ARCHIVE/EXTRACT` 的公开 state 现已附带真实 `progressPercent`，`MEDIA_META` 会暴露 `metadataStage`；多实例下会先回收 lease 过期的 `RUNNING` 任务，再领取 `QUEUED` 任务，旧 worker 若丢失 owner 则不会再覆盖新状态。 - 2026-04-09 桌面端 `Files` 已补最近 10 条后台任务面板，支持查看状态、取消 `QUEUED/RUNNING` 任务，并可为当前选中文件创建媒体信息提取任务；移动端和 archive/extract 的前端入口暂未接入。 +- 2026-04-09 files 后端结构清理：`backend/src/main/java/com/yoyuzh/files` 不再平铺大部分领域类，现已按职责重组为 `core/upload/share/search/events/tasks/storage/policy` 八个子包；类名、接口路径、数据库表名/字段名和现有测试语义保持不变，主要是通过 package 重组、import 修正和测试路径同步降低后续继续演进 upload/share/search/events/tasks/storage-policy 的维护摩擦。