refactor(files): reorganize backend package layout
This commit is contained in:
@@ -9,13 +9,13 @@ import com.yoyuzh.auth.RefreshTokenService;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.ErrorCode;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.FileBlobRepository;
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.StoragePolicy;
|
||||
import com.yoyuzh.files.StoragePolicyRepository;
|
||||
import com.yoyuzh.files.StoragePolicyService;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicy;
|
||||
import com.yoyuzh.files.policy.StoragePolicyRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.transfer.OfflineTransferSessionRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.domain.Page;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.yoyuzh.admin;
|
||||
|
||||
import com.yoyuzh.files.StoragePolicyCapabilities;
|
||||
import com.yoyuzh.files.StoragePolicyCredentialMode;
|
||||
import com.yoyuzh.files.StoragePolicyType;
|
||||
import com.yoyuzh.files.policy.StoragePolicyCapabilities;
|
||||
import com.yoyuzh.files.policy.StoragePolicyCredentialMode;
|
||||
import com.yoyuzh.files.policy.StoragePolicyType;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ package com.yoyuzh.api.v2.files;
|
||||
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.FileEventService;
|
||||
import com.yoyuzh.files.events.FileEventService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.security.core.annotation.AuthenticationPrincipal;
|
||||
import org.springframework.security.core.userdetails.UserDetails;
|
||||
|
||||
@@ -6,9 +6,9 @@ import com.yoyuzh.api.v2.ApiV2Response;
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.FileMetadataResponse;
|
||||
import com.yoyuzh.files.FileSearchQuery;
|
||||
import com.yoyuzh.files.FileSearchService;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.search.FileSearchQuery;
|
||||
import com.yoyuzh.files.search.FileSearchService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.format.annotation.DateTimeFormat;
|
||||
import org.springframework.security.core.annotation.AuthenticationPrincipal;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.yoyuzh.api.v2.files;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public record PreparedUploadV2Response(
|
||||
boolean direct,
|
||||
String uploadUrl,
|
||||
String method,
|
||||
Map<String, String> headers,
|
||||
String storageName
|
||||
) {
|
||||
}
|
||||
@@ -3,10 +3,11 @@ package com.yoyuzh.api.v2.files;
|
||||
import com.yoyuzh.api.v2.ApiV2Response;
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.UploadSession;
|
||||
import com.yoyuzh.files.UploadSessionCreateCommand;
|
||||
import com.yoyuzh.files.UploadSessionPartCommand;
|
||||
import com.yoyuzh.files.UploadSessionService;
|
||||
import com.yoyuzh.files.upload.UploadSession;
|
||||
import com.yoyuzh.files.upload.UploadSessionCreateCommand;
|
||||
import com.yoyuzh.files.upload.UploadSessionPartCommand;
|
||||
import com.yoyuzh.files.upload.UploadSessionService;
|
||||
import com.yoyuzh.files.storage.PreparedUpload;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.security.core.annotation.AuthenticationPrincipal;
|
||||
@@ -77,10 +78,26 @@ public class UploadSessionV2Controller {
|
||||
return ApiV2Response.success(toResponse(session));
|
||||
}
|
||||
|
||||
@GetMapping("/{sessionId}/parts/{partIndex}/prepare")
|
||||
public ApiV2Response<PreparedUploadV2Response> preparePartUpload(@AuthenticationPrincipal UserDetails userDetails,
|
||||
@PathVariable String sessionId,
|
||||
@PathVariable int partIndex) {
|
||||
User user = userDetailsService.loadDomainUser(userDetails.getUsername());
|
||||
PreparedUpload preparedUpload = uploadSessionService.prepareOwnedPartUpload(user, sessionId, partIndex);
|
||||
return ApiV2Response.success(new PreparedUploadV2Response(
|
||||
preparedUpload.direct(),
|
||||
preparedUpload.uploadUrl(),
|
||||
preparedUpload.method(),
|
||||
preparedUpload.headers(),
|
||||
preparedUpload.storageName()
|
||||
));
|
||||
}
|
||||
|
||||
private UploadSessionV2Response toResponse(UploadSession session) {
|
||||
return new UploadSessionV2Response(
|
||||
session.getSessionId(),
|
||||
session.getObjectKey(),
|
||||
session.getMultipartUploadId() != null,
|
||||
session.getTargetPath(),
|
||||
session.getFilename(),
|
||||
session.getContentType(),
|
||||
|
||||
@@ -5,6 +5,7 @@ import java.time.LocalDateTime;
|
||||
public record UploadSessionV2Response(
|
||||
String sessionId,
|
||||
String objectKey,
|
||||
boolean multipartUpload,
|
||||
String path,
|
||||
String filename,
|
||||
String contentType,
|
||||
|
||||
@@ -4,14 +4,15 @@ import com.yoyuzh.api.v2.ApiV2Response;
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.FileMetadataResponse;
|
||||
import com.yoyuzh.files.ShareV2Service;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.share.ShareV2Service;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Max;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.security.core.annotation.AuthenticationPrincipal;
|
||||
import org.springframework.security.core.userdetails.UserDetails;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
@@ -43,6 +44,12 @@ public class ShareV2Controller {
|
||||
return ApiV2Response.success(shareV2Service.getShare(token));
|
||||
}
|
||||
|
||||
@GetMapping(value = "/{token}", params = "download")
|
||||
public ResponseEntity<?> downloadShare(@PathVariable String token,
|
||||
@RequestParam(required = false) String password) {
|
||||
return shareV2Service.downloadSharedFile(token, password);
|
||||
}
|
||||
|
||||
@PostMapping("/{token}/verify-password")
|
||||
public ApiV2Response<ShareV2Response> verifyPassword(@PathVariable String token,
|
||||
@Valid @RequestBody VerifySharePasswordV2Request request) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.yoyuzh.api.v2.shares;
|
||||
|
||||
import com.yoyuzh.files.FileMetadataResponse;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.yoyuzh.api.v2.tasks;
|
||||
|
||||
import com.yoyuzh.files.BackgroundTaskStatus;
|
||||
import com.yoyuzh.files.BackgroundTaskType;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskStatus;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskType;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
|
||||
@@ -4,9 +4,9 @@ import com.yoyuzh.api.v2.ApiV2Response;
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.BackgroundTask;
|
||||
import com.yoyuzh.files.BackgroundTaskService;
|
||||
import com.yoyuzh.files.BackgroundTaskType;
|
||||
import com.yoyuzh.files.tasks.BackgroundTask;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskService;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskType;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Max;
|
||||
import jakarta.validation.constraints.Min;
|
||||
@@ -63,6 +63,13 @@ public class BackgroundTaskV2Controller {
|
||||
return ApiV2Response.success(toResponse(backgroundTaskService.cancelOwnedTask(user, id)));
|
||||
}
|
||||
|
||||
@PostMapping("/{id}/retry")
|
||||
public ApiV2Response<BackgroundTaskResponse> retry(@AuthenticationPrincipal UserDetails userDetails,
|
||||
@PathVariable Long id) {
|
||||
User user = userDetailsService.loadDomainUser(userDetails.getUsername());
|
||||
return ApiV2Response.success(toResponse(backgroundTaskService.retryOwnedTask(user, id)));
|
||||
}
|
||||
|
||||
@PostMapping("/archive")
|
||||
public ApiV2Response<BackgroundTaskResponse> createArchiveTask(@AuthenticationPrincipal UserDetails userDetails,
|
||||
@Valid @RequestBody CreateBackgroundTaskRequest request) {
|
||||
|
||||
@@ -9,8 +9,8 @@ import com.yoyuzh.auth.dto.UpdateUserProfileRequest;
|
||||
import com.yoyuzh.auth.dto.UserProfileResponse;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.ErrorCode;
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.InitiateUploadResponse;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.upload.InitiateUploadResponse;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.yoyuzh.auth;
|
||||
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.context.annotation.Profile;
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
public interface BackgroundTaskHandler {
|
||||
|
||||
boolean supports(BackgroundTaskType type);
|
||||
|
||||
BackgroundTaskHandlerResult handle(BackgroundTask task);
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface BackgroundTaskRepository extends JpaRepository<BackgroundTask, Long> {
|
||||
|
||||
Page<BackgroundTask> findByUserIdOrderByCreatedAtDesc(Long userId, Pageable pageable);
|
||||
|
||||
Optional<BackgroundTask> findByIdAndUserId(Long id, Long userId);
|
||||
|
||||
List<BackgroundTask> findByStatusOrderByCreatedAtAsc(BackgroundTaskStatus status, Pageable pageable);
|
||||
|
||||
@Modifying
|
||||
@Query("""
|
||||
update BackgroundTask task
|
||||
set task.status = :runningStatus,
|
||||
task.errorMessage = null,
|
||||
task.updatedAt = :updatedAt
|
||||
where task.id = :id
|
||||
and task.status = :queuedStatus
|
||||
""")
|
||||
int claimQueuedTask(@Param("id") Long id,
|
||||
@Param("queuedStatus") BackgroundTaskStatus queuedStatus,
|
||||
@Param("runningStatus") BackgroundTaskStatus runningStatus,
|
||||
@Param("updatedAt") LocalDateTime updatedAt);
|
||||
}
|
||||
@@ -1,314 +0,0 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.api.v2.ApiV2ErrorCode;
|
||||
import com.yoyuzh.api.v2.ApiV2Exception;
|
||||
import com.yoyuzh.auth.User;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class BackgroundTaskService {
|
||||
|
||||
private static final List<String> ARCHIVE_EXTENSIONS = List.of(
|
||||
".zip", ".jar", ".war", ".7z", ".rar", ".tar", ".gz", ".tgz", ".bz2", ".xz"
|
||||
);
|
||||
private static final List<String> MEDIA_EXTENSIONS = List.of(
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg",
|
||||
".mp4", ".mov", ".mkv", ".webm", ".avi",
|
||||
".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"
|
||||
);
|
||||
|
||||
private final BackgroundTaskRepository backgroundTaskRepository;
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask createQueuedFileTask(User user,
|
||||
BackgroundTaskType type,
|
||||
Long fileId,
|
||||
String requestedPath,
|
||||
String correlationId) {
|
||||
StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, user.getId())
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "file not found"));
|
||||
String logicalPath = buildLogicalPath(file);
|
||||
if (!logicalPath.equals(normalizeLogicalPath(requestedPath))) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task path does not match file path");
|
||||
}
|
||||
validateTaskTarget(type, file);
|
||||
|
||||
Map<String, Object> publicState = fileState(file, logicalPath);
|
||||
Map<String, Object> privateState = new LinkedHashMap<>(publicState);
|
||||
privateState.put("taskType", type.name());
|
||||
return createQueuedTask(user, type, publicState, privateState, correlationId);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask createQueuedTask(User user,
|
||||
BackgroundTaskType type,
|
||||
Map<String, Object> publicState,
|
||||
Map<String, Object> privateState,
|
||||
String correlationId) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setUserId(user.getId());
|
||||
task.setType(type);
|
||||
task.setStatus(BackgroundTaskStatus.QUEUED);
|
||||
task.setPublicStateJson(toJson(publicState));
|
||||
task.setPrivateStateJson(toJson(privateState));
|
||||
task.setCorrelationId(normalizeCorrelationId(correlationId));
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
public Page<BackgroundTask> listOwnedTasks(User user, Pageable pageable) {
|
||||
return backgroundTaskRepository.findByUserIdOrderByCreatedAtDesc(user.getId(), pageable);
|
||||
}
|
||||
|
||||
public BackgroundTask getOwnedTask(User user, Long id) {
|
||||
return backgroundTaskRepository.findByIdAndUserId(id, user.getId())
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask cancelOwnedTask(User user, Long id) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
|
||||
if (task.getStatus() == BackgroundTaskStatus.QUEUED || task.getStatus() == BackgroundTaskStatus.RUNNING) {
|
||||
task.setStatus(BackgroundTaskStatus.CANCELLED);
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(null);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
return task;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markRunning(User user, Long id) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
task.setStatus(BackgroundTaskStatus.RUNNING);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markCompleted(User user, Long id) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
task.setStatus(BackgroundTaskStatus.COMPLETED);
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(null);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markFailed(User user, Long id, String errorMessage) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
task.setStatus(BackgroundTaskStatus.FAILED);
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed");
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
public List<Long> findQueuedTaskIds(int limit) {
|
||||
if (limit <= 0) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
return backgroundTaskRepository.findByStatusOrderByCreatedAtAsc(
|
||||
BackgroundTaskStatus.QUEUED,
|
||||
PageRequest.of(0, limit)
|
||||
)
|
||||
.stream()
|
||||
.map(BackgroundTask::getId)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Optional<BackgroundTask> claimQueuedTask(Long id) {
|
||||
int claimed = backgroundTaskRepository.claimQueuedTask(
|
||||
id,
|
||||
BackgroundTaskStatus.QUEUED,
|
||||
BackgroundTaskStatus.RUNNING,
|
||||
LocalDateTime.now()
|
||||
);
|
||||
if (claimed != 1) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return backgroundTaskRepository.findById(id);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markWorkerTaskCompleted(Long id, Map<String, Object> publicStatePatch) {
|
||||
BackgroundTask task = backgroundTaskRepository.findById(id)
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
|
||||
if (task.isTerminal() || task.getStatus() != BackgroundTaskStatus.RUNNING) {
|
||||
return task;
|
||||
}
|
||||
|
||||
Map<String, Object> nextPublicState = parseJsonObject(task.getPublicStateJson());
|
||||
if (publicStatePatch != null) {
|
||||
nextPublicState.putAll(publicStatePatch);
|
||||
}
|
||||
task.setPublicStateJson(toJson(nextPublicState));
|
||||
task.setStatus(BackgroundTaskStatus.COMPLETED);
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(null);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markWorkerTaskFailed(Long id, String errorMessage) {
|
||||
BackgroundTask task = backgroundTaskRepository.findById(id)
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
|
||||
task.setStatus(BackgroundTaskStatus.FAILED);
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed");
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
private String normalizeCorrelationId(String correlationId) {
|
||||
if (StringUtils.hasText(correlationId)) {
|
||||
return correlationId.trim();
|
||||
}
|
||||
return UUID.randomUUID().toString().replace("-", "");
|
||||
}
|
||||
|
||||
private void validateTaskTarget(BackgroundTaskType type, StoredFile file) {
|
||||
if (type == BackgroundTaskType.ARCHIVE) {
|
||||
return;
|
||||
}
|
||||
if (file.isDirectory()) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task target type is not supported");
|
||||
}
|
||||
if (type == BackgroundTaskType.EXTRACT && !isArchiveLike(file)) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "extract task only supports archive files");
|
||||
}
|
||||
if (type == BackgroundTaskType.MEDIA_META && !isMediaLike(file)) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "media metadata task only supports media files");
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> fileState(StoredFile file, String logicalPath) {
|
||||
Map<String, Object> state = new LinkedHashMap<>();
|
||||
state.put("fileId", file.getId());
|
||||
state.put("path", logicalPath);
|
||||
state.put("filename", file.getFilename());
|
||||
state.put("directory", file.isDirectory());
|
||||
state.put("contentType", file.getContentType());
|
||||
state.put("size", file.getSize());
|
||||
return state;
|
||||
}
|
||||
|
||||
private boolean isArchiveLike(StoredFile file) {
|
||||
String contentType = normalizeContentType(file.getContentType());
|
||||
if (contentType.contains("zip")
|
||||
|| contentType.contains("x-tar")
|
||||
|| contentType.contains("gzip")
|
||||
|| contentType.contains("x-7z")
|
||||
|| contentType.contains("x-rar")
|
||||
|| contentType.contains("java-archive")) {
|
||||
return true;
|
||||
}
|
||||
return hasExtension(file.getFilename(), ARCHIVE_EXTENSIONS);
|
||||
}
|
||||
|
||||
private boolean isMediaLike(StoredFile file) {
|
||||
String contentType = normalizeContentType(file.getContentType());
|
||||
if (contentType.startsWith("image/") || contentType.startsWith("video/") || contentType.startsWith("audio/")) {
|
||||
return true;
|
||||
}
|
||||
return hasExtension(file.getFilename(), MEDIA_EXTENSIONS);
|
||||
}
|
||||
|
||||
private boolean hasExtension(String filename, List<String> extensions) {
|
||||
if (!StringUtils.hasText(filename)) {
|
||||
return false;
|
||||
}
|
||||
String normalized = filename.toLowerCase(Locale.ROOT);
|
||||
return extensions.stream().anyMatch(normalized::endsWith);
|
||||
}
|
||||
|
||||
private String normalizeContentType(String contentType) {
|
||||
if (!StringUtils.hasText(contentType)) {
|
||||
return "";
|
||||
}
|
||||
return contentType.trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String buildLogicalPath(StoredFile file) {
|
||||
String parent = normalizeLogicalPath(file.getPath());
|
||||
if ("/".equals(parent)) {
|
||||
return "/" + file.getFilename();
|
||||
}
|
||||
return parent + "/" + file.getFilename();
|
||||
}
|
||||
|
||||
private String normalizeLogicalPath(String path) {
|
||||
if (!StringUtils.hasText(path)) {
|
||||
return "/";
|
||||
}
|
||||
String normalized = path.trim().replace('\\', '/');
|
||||
while (normalized.contains("//")) {
|
||||
normalized = normalized.replace("//", "/");
|
||||
}
|
||||
if (!normalized.startsWith("/")) {
|
||||
normalized = "/" + normalized;
|
||||
}
|
||||
while (normalized.length() > 1 && normalized.endsWith("/")) {
|
||||
normalized = normalized.substring(0, normalized.length() - 1);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String toJson(Map<String, Object> value) {
|
||||
Map<String, Object> safeValue = value == null ? new LinkedHashMap<>() : new LinkedHashMap<>(value);
|
||||
try {
|
||||
return objectMapper.writeValueAsString(safeValue);
|
||||
} catch (JsonProcessingException ex) {
|
||||
throw new IllegalStateException("Failed to serialize background task state", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> parseJsonObject(String value) {
|
||||
if (!StringUtils.hasText(value)) {
|
||||
return new LinkedHashMap<>();
|
||||
}
|
||||
|
||||
try {
|
||||
return objectMapper.readValue(value, new TypeReference<LinkedHashMap<String, Object>>() {
|
||||
});
|
||||
} catch (JsonProcessingException ex) {
|
||||
throw new IllegalStateException("Failed to parse background task state", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
public class BackgroundTaskWorker {
|
||||
|
||||
private static final int DEFAULT_BATCH_SIZE = 5;
|
||||
|
||||
private final BackgroundTaskService backgroundTaskService;
|
||||
private final List<BackgroundTaskHandler> handlers;
|
||||
|
||||
public BackgroundTaskWorker(BackgroundTaskService backgroundTaskService,
|
||||
List<BackgroundTaskHandler> handlers) {
|
||||
this.backgroundTaskService = backgroundTaskService;
|
||||
this.handlers = List.copyOf(handlers);
|
||||
}
|
||||
|
||||
@Scheduled(
|
||||
fixedDelayString = "${app.background-tasks.worker.fixed-delay-ms:30000}",
|
||||
initialDelayString = "${app.background-tasks.worker.initial-delay-ms:30000}"
|
||||
)
|
||||
public void runScheduledBatch() {
|
||||
processQueuedTasks(DEFAULT_BATCH_SIZE);
|
||||
}
|
||||
|
||||
public int processQueuedTasks(int maxTasks) {
|
||||
int processedCount = 0;
|
||||
for (Long taskId : backgroundTaskService.findQueuedTaskIds(maxTasks)) {
|
||||
var claimedTask = backgroundTaskService.claimQueuedTask(taskId);
|
||||
if (claimedTask.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
execute(claimedTask.get());
|
||||
processedCount += 1;
|
||||
}
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
private void execute(BackgroundTask task) {
|
||||
try {
|
||||
BackgroundTaskHandler handler = findHandler(task);
|
||||
BackgroundTaskHandlerResult result = handler.handle(task);
|
||||
backgroundTaskService.markWorkerTaskCompleted(task.getId(), result.publicStatePatch());
|
||||
} catch (Exception ex) {
|
||||
String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage();
|
||||
backgroundTaskService.markWorkerTaskFailed(task.getId(), message);
|
||||
}
|
||||
}
|
||||
|
||||
private BackgroundTaskHandler findHandler(BackgroundTask task) {
|
||||
return handlers.stream()
|
||||
.filter(handler -> handler.supports(task.getType()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IllegalStateException("No background task handler for " + task.getType()));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
public record DownloadUrlResponse(String url) {
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
@@ -1,8 +1,14 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.common.ApiResponse;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.share.CreateFileShareLinkResponse;
|
||||
import com.yoyuzh.files.share.FileShareDetailsResponse;
|
||||
import com.yoyuzh.files.share.ImportSharedFileRequest;
|
||||
import com.yoyuzh.files.upload.CompleteUploadRequest;
|
||||
import com.yoyuzh.files.upload.InitiateUploadRequest;
|
||||
import com.yoyuzh.files.upload.InitiateUploadResponse;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import jakarta.persistence.Column;
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.core.annotation.Order;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
public enum FileEntityType {
|
||||
VERSION,
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.admin.AdminMetricsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
@@ -6,8 +6,18 @@ import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.ErrorCode;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.events.FileEventService;
|
||||
import com.yoyuzh.files.events.FileEventType;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.files.share.CreateFileShareLinkResponse;
|
||||
import com.yoyuzh.files.share.FileShareDetailsResponse;
|
||||
import com.yoyuzh.files.share.FileShareLink;
|
||||
import com.yoyuzh.files.share.FileShareLinkRepository;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import com.yoyuzh.files.storage.PreparedUpload;
|
||||
import com.yoyuzh.files.upload.CompleteUploadRequest;
|
||||
import com.yoyuzh.files.upload.InitiateUploadRequest;
|
||||
import com.yoyuzh.files.upload.InitiateUploadResponse;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
@@ -20,6 +30,7 @@ import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
@@ -41,6 +52,7 @@ import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.Locale;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@Service
|
||||
@@ -566,36 +578,49 @@ public class FileService {
|
||||
});
|
||||
}
|
||||
|
||||
private ResponseEntity<byte[]> downloadDirectory(User user, StoredFile directory) {
|
||||
String logicalPath = buildLogicalPath(directory);
|
||||
String archiveName = directory.getFilename() + ".zip";
|
||||
List<StoredFile> descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(user.getId(), logicalPath)
|
||||
.stream()
|
||||
.sorted(Comparator.comparing(StoredFile::getPath).thenComparing(StoredFile::getFilename))
|
||||
.toList();
|
||||
@Transactional
|
||||
public void importExternalFilesAtomically(User recipient,
|
||||
List<String> directories,
|
||||
List<ExternalFileImport> files) {
|
||||
importExternalFilesAtomically(recipient, directories, files, null);
|
||||
}
|
||||
|
||||
byte[] archiveBytes;
|
||||
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) {
|
||||
Set<String> createdEntries = new LinkedHashSet<>();
|
||||
writeDirectoryEntry(zipOutputStream, createdEntries, directory.getFilename() + "/");
|
||||
@Transactional
|
||||
public void importExternalFilesAtomically(User recipient,
|
||||
List<String> directories,
|
||||
List<ExternalFileImport> files,
|
||||
ExternalImportProgressListener progressListener) {
|
||||
List<String> normalizedDirectories = normalizeExternalImportDirectories(directories);
|
||||
List<ExternalFileImport> normalizedFiles = normalizeExternalImportFiles(files);
|
||||
validateExternalImportBatch(recipient, normalizedDirectories, normalizedFiles);
|
||||
|
||||
for (StoredFile descendant : descendants) {
|
||||
String entryName = buildZipEntryName(directory.getFilename(), logicalPath, descendant);
|
||||
if (descendant.isDirectory()) {
|
||||
writeDirectoryEntry(zipOutputStream, createdEntries, entryName + "/");
|
||||
continue;
|
||||
}
|
||||
|
||||
ensureParentDirectoryEntries(zipOutputStream, createdEntries, entryName);
|
||||
writeFileEntry(zipOutputStream, createdEntries, entryName,
|
||||
fileContentStorage.readBlob(getRequiredBlob(descendant).getObjectKey()));
|
||||
List<String> writtenBlobObjectKeys = new ArrayList<>();
|
||||
int totalDirectoryCount = normalizedDirectories.size();
|
||||
int totalFileCount = normalizedFiles.size();
|
||||
int processedDirectoryCount = 0;
|
||||
int processedFileCount = 0;
|
||||
try {
|
||||
for (String directory : normalizedDirectories) {
|
||||
mkdir(recipient, directory);
|
||||
processedDirectoryCount += 1;
|
||||
reportExternalImportProgress(progressListener, processedFileCount, totalFileCount,
|
||||
processedDirectoryCount, totalDirectoryCount);
|
||||
}
|
||||
zipOutputStream.finish();
|
||||
archiveBytes = outputStream.toByteArray();
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "目录压缩失败");
|
||||
for (ExternalFileImport file : normalizedFiles) {
|
||||
storeExternalImportFile(recipient, file, writtenBlobObjectKeys);
|
||||
processedFileCount += 1;
|
||||
reportExternalImportProgress(progressListener, processedFileCount, totalFileCount,
|
||||
processedDirectoryCount, totalDirectoryCount);
|
||||
}
|
||||
} catch (RuntimeException ex) {
|
||||
cleanupWrittenBlobs(writtenBlobObjectKeys, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
private ResponseEntity<byte[]> downloadDirectory(User user, StoredFile directory) {
|
||||
String archiveName = directory.getFilename() + ".zip";
|
||||
byte[] archiveBytes = buildArchiveBytes(directory);
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION,
|
||||
@@ -604,6 +629,63 @@ public class FileService {
|
||||
.body(archiveBytes);
|
||||
}
|
||||
|
||||
public byte[] buildArchiveBytes(StoredFile source) {
|
||||
return buildArchiveBytes(source, null);
|
||||
}
|
||||
|
||||
public byte[] buildArchiveBytes(StoredFile source, ArchiveBuildProgressListener progressListener) {
|
||||
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) {
|
||||
Set<String> createdEntries = new LinkedHashSet<>();
|
||||
ArchiveBuildProgressState progressState = createArchiveBuildProgressState(source, progressListener);
|
||||
reportArchiveProgress(progressState);
|
||||
if (source.isDirectory()) {
|
||||
writeDirectoryArchiveEntries(zipOutputStream, createdEntries, source, progressState);
|
||||
} else {
|
||||
writeFileArchiveEntry(zipOutputStream, createdEntries, source.getFilename(), source, progressState);
|
||||
}
|
||||
zipOutputStream.finish();
|
||||
return outputStream.toByteArray();
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "目录压缩失败");
|
||||
}
|
||||
}
|
||||
|
||||
public ZipCompatibleArchive readZipCompatibleArchive(StoredFile source) {
|
||||
byte[] archiveBytes = fileContentStorage.readBlob(getRequiredBlob(source).getObjectKey());
|
||||
try (ZipInputStream zipInputStream = new ZipInputStream(
|
||||
new ByteArrayInputStream(archiveBytes),
|
||||
StandardCharsets.UTF_8)) {
|
||||
List<ZipCompatibleArchiveEntry> entries = new ArrayList<>();
|
||||
Map<String, Boolean> seenEntries = new HashMap<>();
|
||||
ZipEntry entry = zipInputStream.getNextEntry();
|
||||
while (entry != null) {
|
||||
String relativePath = normalizeZipCompatibleEntryPath(entry.getName());
|
||||
if (StringUtils.hasText(relativePath)) {
|
||||
boolean directory = entry.isDirectory() || entry.getName().endsWith("/");
|
||||
Boolean existingType = seenEntries.putIfAbsent(relativePath, directory);
|
||||
if (existingType != null) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法");
|
||||
}
|
||||
entries.add(new ZipCompatibleArchiveEntry(
|
||||
relativePath,
|
||||
directory,
|
||||
directory ? new byte[0] : zipInputStream.readAllBytes()
|
||||
));
|
||||
}
|
||||
entry = zipInputStream.getNextEntry();
|
||||
}
|
||||
if (entries.isEmpty() && !hasZipCompatibleSignature(archiveBytes)) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包读取失败");
|
||||
}
|
||||
return new ZipCompatibleArchive(entries, detectCommonRootDirectoryName(entries));
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包读取失败");
|
||||
}
|
||||
}
|
||||
|
||||
private boolean shouldUsePublicPackageDownload(StoredFile storedFile) {
|
||||
return fileContentStorage.supportsDirectDownload()
|
||||
&& StringUtils.hasText(packageDownloadBaseUrl)
|
||||
@@ -854,6 +936,62 @@ public class FileService {
|
||||
ensureWithinStorageQuota(user, size);
|
||||
}
|
||||
|
||||
private List<String> normalizeExternalImportDirectories(List<String> directories) {
|
||||
if (directories == null || directories.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
return directories.stream()
|
||||
.map(this::normalizeDirectoryPath)
|
||||
.distinct()
|
||||
.sorted(Comparator.comparingInt(String::length).thenComparing(String::compareTo))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private List<ExternalFileImport> normalizeExternalImportFiles(List<ExternalFileImport> files) {
|
||||
if (files == null || files.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
return files.stream()
|
||||
.map(file -> new ExternalFileImport(
|
||||
normalizeDirectoryPath(file.path()),
|
||||
normalizeLeafName(file.filename()),
|
||||
StringUtils.hasText(file.contentType()) ? file.contentType().trim() : "application/octet-stream",
|
||||
file.content() == null ? new byte[0] : file.content()
|
||||
))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private void validateExternalImportBatch(User recipient,
|
||||
List<String> directories,
|
||||
List<ExternalFileImport> files) {
|
||||
ensureWithinStorageQuota(recipient, files.stream().mapToLong(ExternalFileImport::size).sum());
|
||||
|
||||
Set<String> plannedTargets = new LinkedHashSet<>();
|
||||
for (String directory : directories) {
|
||||
if ("/".equals(directory)) {
|
||||
continue;
|
||||
}
|
||||
if (!plannedTargets.add(directory)) {
|
||||
continue;
|
||||
}
|
||||
String parentPath = extractParentPath(directory);
|
||||
String directoryName = extractLeafName(directory);
|
||||
if (storedFileRepository.existsByUserIdAndPathAndFilename(recipient.getId(), parentPath, directoryName)) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在");
|
||||
}
|
||||
}
|
||||
|
||||
for (ExternalFileImport file : files) {
|
||||
String logicalPath = buildTargetLogicalPath(file.path(), file.filename());
|
||||
if (plannedTargets.contains(logicalPath) || !plannedTargets.add(logicalPath)) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在");
|
||||
}
|
||||
if (storedFileRepository.existsByUserIdAndPathAndFilename(recipient.getId(), file.path(), file.filename())) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "同目录下文件已存在");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureWithinStorageQuota(User user, long additionalBytes) {
|
||||
if (additionalBytes <= 0) {
|
||||
return;
|
||||
@@ -900,6 +1038,25 @@ public class FileService {
|
||||
}
|
||||
}
|
||||
|
||||
private void storeExternalImportFile(User recipient,
|
||||
ExternalFileImport file,
|
||||
List<String> writtenBlobObjectKeys) {
|
||||
validateUpload(recipient, file.path(), file.filename(), file.size());
|
||||
ensureDirectoryHierarchy(recipient, file.path());
|
||||
String objectKey = createBlobObjectKey();
|
||||
writtenBlobObjectKeys.add(objectKey);
|
||||
fileContentStorage.storeBlob(objectKey, file.contentType(), file.content());
|
||||
FileBlob blob = createAndSaveBlob(objectKey, file.contentType(), file.size());
|
||||
saveFileMetadata(
|
||||
recipient,
|
||||
file.path(),
|
||||
file.filename(),
|
||||
file.contentType(),
|
||||
file.size(),
|
||||
blob
|
||||
);
|
||||
}
|
||||
|
||||
private void moveToRecycleBin(List<StoredFile> filesToRecycle, Long recycleRootId) {
|
||||
if (filesToRecycle.isEmpty()) {
|
||||
return;
|
||||
@@ -1071,6 +1228,37 @@ public class FileService {
|
||||
return savedFile;
|
||||
}
|
||||
|
||||
private void writeDirectoryArchiveEntries(ZipOutputStream zipOutputStream,
|
||||
Set<String> createdEntries,
|
||||
StoredFile directory,
|
||||
ArchiveBuildProgressState progressState) throws IOException {
|
||||
String logicalPath = buildLogicalPath(directory);
|
||||
List<StoredFile> descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(directory.getUser().getId(), logicalPath)
|
||||
.stream()
|
||||
.sorted(Comparator.comparing(StoredFile::getPath).thenComparing(StoredFile::getFilename))
|
||||
.toList();
|
||||
writeDirectoryEntry(zipOutputStream, createdEntries, directory.getFilename() + "/", progressState);
|
||||
|
||||
for (StoredFile descendant : descendants) {
|
||||
String entryName = buildZipEntryName(directory.getFilename(), logicalPath, descendant);
|
||||
if (descendant.isDirectory()) {
|
||||
writeDirectoryEntry(zipOutputStream, createdEntries, entryName + "/", progressState);
|
||||
continue;
|
||||
}
|
||||
writeFileArchiveEntry(zipOutputStream, createdEntries, entryName, descendant, progressState);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeFileArchiveEntry(ZipOutputStream zipOutputStream,
|
||||
Set<String> createdEntries,
|
||||
String entryName,
|
||||
StoredFile file,
|
||||
ArchiveBuildProgressState progressState) throws IOException {
|
||||
ensureParentDirectoryEntries(zipOutputStream, createdEntries, entryName, progressState);
|
||||
writeFileEntry(zipOutputStream, createdEntries, entryName, progressState,
|
||||
fileContentStorage.readBlob(getRequiredBlob(file).getObjectKey()));
|
||||
}
|
||||
|
||||
private String buildZipEntryName(String rootDirectoryName, String rootLogicalPath, StoredFile storedFile) {
|
||||
StringBuilder entryName = new StringBuilder(rootDirectoryName).append('/');
|
||||
if (!storedFile.getPath().equals(rootLogicalPath)) {
|
||||
@@ -1080,24 +1268,153 @@ public class FileService {
|
||||
return entryName.toString();
|
||||
}
|
||||
|
||||
private void ensureParentDirectoryEntries(ZipOutputStream zipOutputStream, Set<String> createdEntries, String entryName) throws IOException {
|
||||
private String normalizeZipCompatibleEntryPath(String entryName) {
|
||||
String normalized = entryName == null ? "" : entryName.trim().replace("\\", "/");
|
||||
if (!StringUtils.hasText(normalized)) {
|
||||
return "";
|
||||
}
|
||||
if (normalized.startsWith("/")) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法");
|
||||
}
|
||||
while (normalized.endsWith("/")) {
|
||||
normalized = normalized.substring(0, normalized.length() - 1);
|
||||
}
|
||||
if (!StringUtils.hasText(normalized)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
StringBuilder sanitized = new StringBuilder();
|
||||
for (String segment : normalized.split("/")) {
|
||||
if (!StringUtils.hasText(segment) || ".".equals(segment) || "..".equals(segment)) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法");
|
||||
}
|
||||
if (sanitized.length() > 0) {
|
||||
sanitized.append('/');
|
||||
}
|
||||
sanitized.append(normalizeLeafName(segment));
|
||||
}
|
||||
return sanitized.toString();
|
||||
}
|
||||
|
||||
private String detectCommonRootDirectoryName(List<ZipCompatibleArchiveEntry> entries) {
|
||||
String candidate = null;
|
||||
boolean hasNestedEntry = false;
|
||||
boolean hasDirectoryCandidate = false;
|
||||
for (ZipCompatibleArchiveEntry entry : entries) {
|
||||
String relativePath = entry.relativePath();
|
||||
int slashIndex = relativePath.indexOf('/');
|
||||
String topSegment = slashIndex >= 0 ? relativePath.substring(0, slashIndex) : relativePath;
|
||||
if (candidate == null) {
|
||||
candidate = topSegment;
|
||||
} else if (!candidate.equals(topSegment)) {
|
||||
return null;
|
||||
}
|
||||
if (slashIndex >= 0) {
|
||||
hasNestedEntry = true;
|
||||
}
|
||||
if (entry.directory() && candidate.equals(relativePath)) {
|
||||
hasDirectoryCandidate = true;
|
||||
}
|
||||
if (!entry.directory() && candidate.equals(relativePath)) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (!hasNestedEntry && !hasDirectoryCandidate) {
|
||||
return null;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private boolean hasZipCompatibleSignature(byte[] archiveBytes) {
|
||||
if (archiveBytes == null || archiveBytes.length < 4) {
|
||||
return false;
|
||||
}
|
||||
return archiveBytes[0] == 'P'
|
||||
&& archiveBytes[1] == 'K'
|
||||
&& (archiveBytes[2] == 3 || archiveBytes[2] == 5 || archiveBytes[2] == 7)
|
||||
&& (archiveBytes[3] == 4 || archiveBytes[3] == 6 || archiveBytes[3] == 8);
|
||||
}
|
||||
|
||||
public ArchiveSourceSummary summarizeArchiveSource(StoredFile source) {
|
||||
if (!source.isDirectory()) {
|
||||
return new ArchiveSourceSummary(1, 0);
|
||||
}
|
||||
String logicalPath = buildLogicalPath(source);
|
||||
List<StoredFile> descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(source.getUser().getId(), logicalPath);
|
||||
int directoryCount = 1 + (int) descendants.stream().filter(StoredFile::isDirectory).count();
|
||||
int fileCount = (int) descendants.stream().filter(file -> !file.isDirectory()).count();
|
||||
return new ArchiveSourceSummary(fileCount, directoryCount);
|
||||
}
|
||||
|
||||
private ArchiveBuildProgressState createArchiveBuildProgressState(StoredFile source,
|
||||
ArchiveBuildProgressListener progressListener) {
|
||||
if (progressListener == null) {
|
||||
return null;
|
||||
}
|
||||
ArchiveSourceSummary summary = summarizeArchiveSource(source);
|
||||
return new ArchiveBuildProgressState(progressListener, summary.fileCount(), summary.directoryCount());
|
||||
}
|
||||
|
||||
private void reportArchiveProgress(ArchiveBuildProgressState progressState) {
|
||||
if (progressState == null) {
|
||||
return;
|
||||
}
|
||||
progressState.listener.onProgress(new ArchiveBuildProgress(
|
||||
progressState.processedFileCount,
|
||||
progressState.totalFileCount,
|
||||
progressState.processedDirectoryCount,
|
||||
progressState.totalDirectoryCount
|
||||
));
|
||||
}
|
||||
|
||||
private void reportExternalImportProgress(ExternalImportProgressListener progressListener,
|
||||
int processedFileCount,
|
||||
int totalFileCount,
|
||||
int processedDirectoryCount,
|
||||
int totalDirectoryCount) {
|
||||
if (progressListener == null) {
|
||||
return;
|
||||
}
|
||||
progressListener.onProgress(new ExternalImportProgress(
|
||||
processedFileCount,
|
||||
totalFileCount,
|
||||
processedDirectoryCount,
|
||||
totalDirectoryCount
|
||||
));
|
||||
}
|
||||
|
||||
private void ensureParentDirectoryEntries(ZipOutputStream zipOutputStream,
|
||||
Set<String> createdEntries,
|
||||
String entryName,
|
||||
ArchiveBuildProgressState progressState) throws IOException {
|
||||
int slashIndex = entryName.indexOf('/');
|
||||
while (slashIndex >= 0) {
|
||||
writeDirectoryEntry(zipOutputStream, createdEntries, entryName.substring(0, slashIndex + 1));
|
||||
writeDirectoryEntry(zipOutputStream, createdEntries, entryName.substring(0, slashIndex + 1), progressState);
|
||||
slashIndex = entryName.indexOf('/', slashIndex + 1);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeDirectoryEntry(ZipOutputStream zipOutputStream, Set<String> createdEntries, String entryName) throws IOException {
|
||||
private void writeDirectoryEntry(ZipOutputStream zipOutputStream,
|
||||
Set<String> createdEntries,
|
||||
String entryName,
|
||||
ArchiveBuildProgressState progressState) throws IOException {
|
||||
if (!createdEntries.add(entryName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
zipOutputStream.putNextEntry(new ZipEntry(entryName));
|
||||
zipOutputStream.closeEntry();
|
||||
if (progressState != null) {
|
||||
progressState.processedDirectoryCount += 1;
|
||||
reportArchiveProgress(progressState);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeFileEntry(ZipOutputStream zipOutputStream, Set<String> createdEntries, String entryName, byte[] content)
|
||||
private void writeFileEntry(ZipOutputStream zipOutputStream,
|
||||
Set<String> createdEntries,
|
||||
String entryName,
|
||||
ArchiveBuildProgressState progressState,
|
||||
byte[] content)
|
||||
throws IOException {
|
||||
if (!createdEntries.add(entryName)) {
|
||||
return;
|
||||
@@ -1106,6 +1423,10 @@ public class FileService {
|
||||
zipOutputStream.putNextEntry(new ZipEntry(entryName));
|
||||
zipOutputStream.write(content);
|
||||
zipOutputStream.closeEntry();
|
||||
if (progressState != null) {
|
||||
progressState.processedFileCount += 1;
|
||||
reportArchiveProgress(progressState);
|
||||
}
|
||||
}
|
||||
|
||||
private void recordFileEvent(User user,
|
||||
@@ -1170,6 +1491,16 @@ public class FileService {
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupWrittenBlobs(List<String> writtenBlobObjectKeys, RuntimeException ex) {
|
||||
for (String objectKey : writtenBlobObjectKeys) {
|
||||
try {
|
||||
fileContentStorage.deleteBlob(objectKey);
|
||||
} catch (RuntimeException cleanupEx) {
|
||||
ex.addSuppressed(cleanupEx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private FileBlob createAndSaveBlob(String objectKey, String contentType, long size) {
|
||||
FileBlob blob = new FileBlob();
|
||||
blob.setObjectKey(objectKey);
|
||||
@@ -1248,4 +1579,57 @@ public class FileService {
|
||||
private interface BlobWriteOperation<T> {
|
||||
T run();
|
||||
}
|
||||
|
||||
public static record ZipCompatibleArchive(List<ZipCompatibleArchiveEntry> entries, String commonRootDirectoryName) {
|
||||
}
|
||||
|
||||
public static record ZipCompatibleArchiveEntry(String relativePath, boolean directory, byte[] content) {
|
||||
}
|
||||
|
||||
public static record ExternalFileImport(String path, String filename, String contentType, byte[] content) {
|
||||
public long size() {
|
||||
return content == null ? 0L : content.length;
|
||||
}
|
||||
}
|
||||
|
||||
public record ArchiveSourceSummary(int fileCount, int directoryCount) {
|
||||
}
|
||||
|
||||
public record ArchiveBuildProgress(int processedFileCount,
|
||||
int totalFileCount,
|
||||
int processedDirectoryCount,
|
||||
int totalDirectoryCount) {
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ArchiveBuildProgressListener {
|
||||
void onProgress(ArchiveBuildProgress progress);
|
||||
}
|
||||
|
||||
public record ExternalImportProgress(int processedFileCount,
|
||||
int totalFileCount,
|
||||
int processedDirectoryCount,
|
||||
int totalDirectoryCount) {
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ExternalImportProgressListener {
|
||||
void onProgress(ExternalImportProgress progress);
|
||||
}
|
||||
|
||||
private static final class ArchiveBuildProgressState {
|
||||
private final ArchiveBuildProgressListener listener;
|
||||
private final int totalFileCount;
|
||||
private final int totalDirectoryCount;
|
||||
private int processedFileCount;
|
||||
private int processedDirectoryCount;
|
||||
|
||||
private ArchiveBuildProgressState(ArchiveBuildProgressListener listener,
|
||||
int totalFileCount,
|
||||
int totalDirectoryCount) {
|
||||
this.listener = listener;
|
||||
this.totalFileCount = totalFileCount;
|
||||
this.totalDirectoryCount = totalDirectoryCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import jakarta.persistence.Column;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.events;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.events;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.events;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.events;
|
||||
|
||||
public enum FileEventType {
|
||||
CREATED,
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.policy;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.policy;
|
||||
|
||||
public record StoragePolicyCapabilities(
|
||||
boolean directUpload,
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.policy;
|
||||
|
||||
public enum StoragePolicyCredentialMode {
|
||||
NONE,
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.policy;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.policy;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
@@ -57,7 +57,7 @@ public class StoragePolicyService implements CommandLineRunner {
|
||||
policy.setMaxSizeBytes(properties.getMaxFileSize());
|
||||
policy.setCapabilitiesJson(writeCapabilities(new StoragePolicyCapabilities(
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.policy;
|
||||
|
||||
public enum StoragePolicyType {
|
||||
LOCAL,
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.search;
|
||||
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.FetchType;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.search;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.search;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.search;
|
||||
|
||||
import com.yoyuzh.api.v2.ApiV2ErrorCode;
|
||||
import com.yoyuzh.api.v2.ApiV2Exception;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.share;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.share;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.share;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.FetchType;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.share;
|
||||
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.share;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.share;
|
||||
|
||||
import com.yoyuzh.api.v2.ApiV2ErrorCode;
|
||||
import com.yoyuzh.api.v2.ApiV2Exception;
|
||||
@@ -7,10 +7,15 @@ import com.yoyuzh.api.v2.shares.ImportShareV2Request;
|
||||
import com.yoyuzh.api.v2.shares.ShareV2Response;
|
||||
import com.yoyuzh.api.v2.shares.VerifySharePasswordV2Request;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.security.crypto.password.PasswordEncoder;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.StringUtils;
|
||||
@@ -85,6 +90,17 @@ public class ShareV2Service {
|
||||
return importedFile;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public ResponseEntity<?> downloadSharedFile(String token, String password) {
|
||||
FileShareLink shareLink = getShareLink(token);
|
||||
ensureShareNotExpired(shareLink);
|
||||
ensureDownloadAllowed(shareLink);
|
||||
ensurePasswordAccepted(shareLink, password);
|
||||
|
||||
shareLink.setDownloadCount(shareLink.getDownloadCountOrZero() + 1);
|
||||
return fileService.download(shareLink.getOwner(), shareLink.getFile().getId());
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Page<ShareV2Response> listOwnedShares(User user, Pageable pageable) {
|
||||
return fileShareLinkRepository.findByOwnerIdOrderByCreatedAtDesc(user.getId(), pageable)
|
||||
@@ -114,6 +130,18 @@ public class ShareV2Service {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "import disabled");
|
||||
}
|
||||
|
||||
ensureQuotaAvailable(shareLink);
|
||||
}
|
||||
|
||||
private void ensureDownloadAllowed(FileShareLink shareLink) {
|
||||
if (!shareLink.isAllowDownloadEnabled()) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "download disabled");
|
||||
}
|
||||
|
||||
ensureQuotaAvailable(shareLink);
|
||||
}
|
||||
|
||||
private void ensureQuotaAvailable(FileShareLink shareLink) {
|
||||
Integer maxDownloads = shareLink.getMaxDownloads();
|
||||
if (maxDownloads != null && shareLink.getDownloadCountOrZero() >= maxDownloads) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "share quota exceeded");
|
||||
@@ -48,6 +48,26 @@ public interface FileContentStorage {
|
||||
|
||||
void deleteBlob(String objectKey);
|
||||
|
||||
default String createMultipartUpload(String objectKey, String contentType) {
|
||||
throw new UnsupportedOperationException("Multipart upload is not supported by this storage");
|
||||
}
|
||||
|
||||
default PreparedUpload prepareMultipartPartUpload(String objectKey,
|
||||
String uploadId,
|
||||
int partNumber,
|
||||
String contentType,
|
||||
long size) {
|
||||
throw new UnsupportedOperationException("Multipart upload is not supported by this storage");
|
||||
}
|
||||
|
||||
default void completeMultipartUpload(String objectKey, String uploadId, java.util.List<MultipartCompletedPart> parts) {
|
||||
throw new UnsupportedOperationException("Multipart upload is not supported by this storage");
|
||||
}
|
||||
|
||||
default void abortMultipartUpload(String objectKey, String uploadId) {
|
||||
throw new UnsupportedOperationException("Multipart upload is not supported by this storage");
|
||||
}
|
||||
|
||||
String createBlobDownloadUrl(String objectKey, String filename);
|
||||
|
||||
void createDirectory(Long userId, String logicalPath);
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package com.yoyuzh.files.storage;
|
||||
|
||||
public record MultipartCompletedPart(
|
||||
int partNumber,
|
||||
String etag
|
||||
) {
|
||||
}
|
||||
@@ -9,22 +9,31 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
import software.amazon.awssdk.core.ResponseBytes;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.http.SdkHttpMethod;
|
||||
import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
|
||||
import software.amazon.awssdk.services.s3.model.CompletedPart;
|
||||
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -160,6 +169,90 @@ public class S3FileContentStorage implements FileContentStorage {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String createMultipartUpload(String objectKey, String contentType) {
|
||||
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||
CreateMultipartUploadRequest.Builder requestBuilder = CreateMultipartUploadRequest.builder()
|
||||
.bucket(session.bucket())
|
||||
.key(normalizeObjectKey(objectKey));
|
||||
if (StringUtils.hasText(contentType)) {
|
||||
requestBuilder.contentType(contentType);
|
||||
}
|
||||
try {
|
||||
return session.s3Client().createMultipartUpload(requestBuilder.build()).uploadId();
|
||||
} catch (S3Exception ex) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload init failed");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PreparedUpload prepareMultipartPartUpload(String objectKey,
|
||||
String uploadId,
|
||||
int partNumber,
|
||||
String contentType,
|
||||
long size) {
|
||||
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||
UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
|
||||
.bucket(session.bucket())
|
||||
.key(normalizeObjectKey(objectKey))
|
||||
.uploadId(uploadId)
|
||||
.partNumber(partNumber)
|
||||
.contentLength(size)
|
||||
.build();
|
||||
UploadPartPresignRequest presignRequest = UploadPartPresignRequest.builder()
|
||||
.signatureDuration(Duration.ofSeconds(Math.max(1, properties.getTtlSeconds())))
|
||||
.uploadPartRequest(uploadPartRequest)
|
||||
.build();
|
||||
PresignedUploadPartRequest presignedRequest = session.s3Presigner().presignUploadPart(presignRequest);
|
||||
Map<String, String> headers = flattenSignedHeaders(presignedRequest.signedHeaders());
|
||||
if (StringUtils.hasText(contentType)) {
|
||||
headers.put("Content-Type", contentType);
|
||||
}
|
||||
return new PreparedUpload(
|
||||
true,
|
||||
presignedRequest.url().toString(),
|
||||
resolveUploadMethod(presignedRequest),
|
||||
headers,
|
||||
objectKey
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void completeMultipartUpload(String objectKey, String uploadId, List<MultipartCompletedPart> parts) {
|
||||
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||
List<CompletedPart> completedParts = parts.stream()
|
||||
.sorted(Comparator.comparingInt(MultipartCompletedPart::partNumber))
|
||||
.map(part -> CompletedPart.builder()
|
||||
.partNumber(part.partNumber())
|
||||
.eTag(part.etag())
|
||||
.build())
|
||||
.toList();
|
||||
try {
|
||||
session.s3Client().completeMultipartUpload(CompleteMultipartUploadRequest.builder()
|
||||
.bucket(session.bucket())
|
||||
.key(normalizeObjectKey(objectKey))
|
||||
.uploadId(uploadId)
|
||||
.multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
|
||||
.build());
|
||||
} catch (S3Exception ex) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload complete failed");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void abortMultipartUpload(String objectKey, String uploadId) {
|
||||
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||
try {
|
||||
session.s3Client().abortMultipartUpload(AbortMultipartUploadRequest.builder()
|
||||
.bucket(session.bucket())
|
||||
.key(normalizeObjectKey(objectKey))
|
||||
.uploadId(uploadId)
|
||||
.build());
|
||||
} catch (S3Exception ex) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload abort failed");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String createBlobDownloadUrl(String objectKey, String filename) {
|
||||
return createDownloadUrl(sessionProvider.currentSession(), normalizeObjectKey(objectKey), filename);
|
||||
@@ -330,6 +423,13 @@ public class S3FileContentStorage implements FileContentStorage {
|
||||
return presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : "POST";
|
||||
}
|
||||
|
||||
private String resolveUploadMethod(PresignedUploadPartRequest presignedRequest) {
|
||||
if (presignedRequest.httpRequest() == null) {
|
||||
return "PUT";
|
||||
}
|
||||
return presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : presignedRequest.httpRequest().method().name();
|
||||
}
|
||||
|
||||
private Map<String, String> resolveUploadHeaders(PresignedPutObjectRequest presignedRequest, String contentType) {
|
||||
Map<String, String> headers = flattenSignedHeaders(presignedRequest.signedHeaders());
|
||||
if (StringUtils.hasText(contentType)) {
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import jakarta.transaction.Transactional;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
@Transactional
|
||||
public class ArchiveBackgroundTaskHandler implements BackgroundTaskHandler {
|
||||
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final UserRepository userRepository;
|
||||
private final FileService fileService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public ArchiveBackgroundTaskHandler(StoredFileRepository storedFileRepository,
|
||||
UserRepository userRepository,
|
||||
FileService fileService,
|
||||
ObjectMapper objectMapper) {
|
||||
this.storedFileRepository = storedFileRepository;
|
||||
this.userRepository = userRepository;
|
||||
this.fileService = fileService;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supports(BackgroundTaskType type) {
|
||||
return type == BackgroundTaskType.ARCHIVE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BackgroundTaskHandlerResult handle(BackgroundTask task) {
|
||||
return handle(task, publicStatePatch -> {
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
|
||||
Map<String, Object> state = parseState(task.getPrivateStateJson(), task.getPublicStateJson());
|
||||
Long fileId = extractLong(state.get("fileId"));
|
||||
String outputPath = extractText(state.get("outputPath"));
|
||||
String outputFilename = extractText(state.get("outputFilename"));
|
||||
if (fileId == null) {
|
||||
throw new IllegalStateException("archive task missing fileId");
|
||||
}
|
||||
if (!StringUtils.hasText(outputPath) || !StringUtils.hasText(outputFilename)) {
|
||||
throw new IllegalStateException("archive task missing output target");
|
||||
}
|
||||
|
||||
StoredFile source = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId())
|
||||
.orElseThrow(() -> new IllegalStateException("archive task file not found"));
|
||||
User user = userRepository.findById(task.getUserId())
|
||||
.orElseThrow(() -> new IllegalStateException("archive task user not found"));
|
||||
|
||||
FileService.ArchiveSourceSummary summary = fileService.summarizeArchiveSource(source);
|
||||
progressReporter.report(progressPatch(0, summary.fileCount(), 0, summary.directoryCount()));
|
||||
byte[] archiveBytes = fileService.buildArchiveBytes(source, progress ->
|
||||
progressReporter.report(progressPatch(
|
||||
progress.processedFileCount(),
|
||||
progress.totalFileCount(),
|
||||
progress.processedDirectoryCount(),
|
||||
progress.totalDirectoryCount()
|
||||
)));
|
||||
FileMetadataResponse archivedFile = fileService.importExternalFile(
|
||||
user,
|
||||
outputPath,
|
||||
outputFilename,
|
||||
"application/zip",
|
||||
archiveBytes.length,
|
||||
archiveBytes
|
||||
);
|
||||
|
||||
Map<String, Object> publicStatePatch = new LinkedHashMap<>();
|
||||
publicStatePatch.put("worker", "archive");
|
||||
publicStatePatch.put("archivedFileId", archivedFile.id());
|
||||
publicStatePatch.put("archivedFilename", archivedFile.filename());
|
||||
publicStatePatch.put("archivedPath", archivedFile.path());
|
||||
publicStatePatch.put("archiveSize", archiveBytes.length);
|
||||
publicStatePatch.putAll(progressPatch(
|
||||
summary.fileCount(),
|
||||
summary.fileCount(),
|
||||
summary.directoryCount(),
|
||||
summary.directoryCount()
|
||||
));
|
||||
return new BackgroundTaskHandlerResult(publicStatePatch);
|
||||
}
|
||||
|
||||
private Map<String, Object> progressPatch(int processedFileCount,
|
||||
int totalFileCount,
|
||||
int processedDirectoryCount,
|
||||
int totalDirectoryCount) {
|
||||
Map<String, Object> patch = new LinkedHashMap<>();
|
||||
patch.put("processedFileCount", processedFileCount);
|
||||
patch.put("totalFileCount", totalFileCount);
|
||||
patch.put("processedDirectoryCount", processedDirectoryCount);
|
||||
patch.put("totalDirectoryCount", totalDirectoryCount);
|
||||
patch.put("progressPercent", calculateProgressPercent(
|
||||
processedFileCount,
|
||||
totalFileCount,
|
||||
processedDirectoryCount,
|
||||
totalDirectoryCount
|
||||
));
|
||||
return patch;
|
||||
}
|
||||
|
||||
private int calculateProgressPercent(int processedFileCount,
|
||||
int totalFileCount,
|
||||
int processedDirectoryCount,
|
||||
int totalDirectoryCount) {
|
||||
int total = Math.max(0, totalFileCount) + Math.max(0, totalDirectoryCount);
|
||||
int processed = Math.max(0, processedFileCount) + Math.max(0, processedDirectoryCount);
|
||||
if (total <= 0) {
|
||||
return 100;
|
||||
}
|
||||
return Math.min(100, (int) Math.floor((processed * 100.0d) / total));
|
||||
}
|
||||
|
||||
private Map<String, Object> parseState(String privateStateJson, String publicStateJson) {
|
||||
Map<String, Object> state = new LinkedHashMap<>(parseJsonObject(publicStateJson));
|
||||
state.putAll(parseJsonObject(privateStateJson));
|
||||
return state;
|
||||
}
|
||||
|
||||
private Map<String, Object> parseJsonObject(String json) {
|
||||
if (!StringUtils.hasText(json)) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
|
||||
});
|
||||
} catch (JsonProcessingException ex) {
|
||||
throw new IllegalStateException("archive task state is invalid", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private Long extractLong(Object value) {
|
||||
if (value instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
if (value instanceof String text && StringUtils.hasText(text)) {
|
||||
return Long.parseLong(text.trim());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractText(Object value) {
|
||||
if (value instanceof String text && StringUtils.hasText(text)) {
|
||||
return text.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
@@ -18,6 +18,7 @@ import java.time.LocalDateTime;
|
||||
@Table(name = "portal_background_task", indexes = {
|
||||
@Index(name = "idx_background_task_user_created_at", columnList = "user_id,created_at"),
|
||||
@Index(name = "idx_background_task_status_created_at", columnList = "status,created_at"),
|
||||
@Index(name = "idx_background_task_status_lease_expires_at", columnList = "status,lease_expires_at"),
|
||||
@Index(name = "idx_background_task_correlation_id", columnList = "correlation_id")
|
||||
})
|
||||
public class BackgroundTask {
|
||||
@@ -49,6 +50,24 @@ public class BackgroundTask {
|
||||
@Column(name = "error_message", length = 512)
|
||||
private String errorMessage;
|
||||
|
||||
@Column(name = "attempt_count", nullable = false)
|
||||
private Integer attemptCount;
|
||||
|
||||
@Column(name = "max_attempts", nullable = false)
|
||||
private Integer maxAttempts;
|
||||
|
||||
@Column(name = "next_run_at")
|
||||
private LocalDateTime nextRunAt;
|
||||
|
||||
@Column(name = "lease_owner", length = 128)
|
||||
private String leaseOwner;
|
||||
|
||||
@Column(name = "lease_expires_at")
|
||||
private LocalDateTime leaseExpiresAt;
|
||||
|
||||
@Column(name = "heartbeat_at")
|
||||
private LocalDateTime heartbeatAt;
|
||||
|
||||
@Column(name = "created_at", nullable = false)
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
@@ -70,6 +89,12 @@ public class BackgroundTask {
|
||||
if (status == null) {
|
||||
status = BackgroundTaskStatus.QUEUED;
|
||||
}
|
||||
if (attemptCount == null) {
|
||||
attemptCount = 0;
|
||||
}
|
||||
if (maxAttempts == null) {
|
||||
maxAttempts = 1;
|
||||
}
|
||||
if (publicStateJson == null) {
|
||||
publicStateJson = "{}";
|
||||
}
|
||||
@@ -147,6 +172,54 @@ public class BackgroundTask {
|
||||
this.errorMessage = errorMessage;
|
||||
}
|
||||
|
||||
public Integer getAttemptCount() {
|
||||
return attemptCount;
|
||||
}
|
||||
|
||||
public void setAttemptCount(Integer attemptCount) {
|
||||
this.attemptCount = attemptCount;
|
||||
}
|
||||
|
||||
public Integer getMaxAttempts() {
|
||||
return maxAttempts;
|
||||
}
|
||||
|
||||
public void setMaxAttempts(Integer maxAttempts) {
|
||||
this.maxAttempts = maxAttempts;
|
||||
}
|
||||
|
||||
public LocalDateTime getNextRunAt() {
|
||||
return nextRunAt;
|
||||
}
|
||||
|
||||
public void setNextRunAt(LocalDateTime nextRunAt) {
|
||||
this.nextRunAt = nextRunAt;
|
||||
}
|
||||
|
||||
public String getLeaseOwner() {
|
||||
return leaseOwner;
|
||||
}
|
||||
|
||||
public void setLeaseOwner(String leaseOwner) {
|
||||
this.leaseOwner = leaseOwner;
|
||||
}
|
||||
|
||||
public LocalDateTime getLeaseExpiresAt() {
|
||||
return leaseExpiresAt;
|
||||
}
|
||||
|
||||
public void setLeaseExpiresAt(LocalDateTime leaseExpiresAt) {
|
||||
this.leaseExpiresAt = leaseExpiresAt;
|
||||
}
|
||||
|
||||
public LocalDateTime getHeartbeatAt() {
|
||||
return heartbeatAt;
|
||||
}
|
||||
|
||||
public void setHeartbeatAt(LocalDateTime heartbeatAt) {
|
||||
this.heartbeatAt = heartbeatAt;
|
||||
}
|
||||
|
||||
public LocalDateTime getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
public enum BackgroundTaskFailureCategory {
|
||||
UNSUPPORTED_INPUT(false),
|
||||
DATA_STATE(false),
|
||||
TRANSIENT_INFRASTRUCTURE(true),
|
||||
RATE_LIMITED(true),
|
||||
UNKNOWN(true);
|
||||
|
||||
private final boolean retryable;
|
||||
|
||||
BackgroundTaskFailureCategory(boolean retryable) {
|
||||
this.retryable = retryable;
|
||||
}
|
||||
|
||||
public boolean isRetryable() {
|
||||
return retryable;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
public interface BackgroundTaskHandler {
|
||||
|
||||
boolean supports(BackgroundTaskType type);
|
||||
|
||||
BackgroundTaskHandlerResult handle(BackgroundTask task);
|
||||
|
||||
default BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
|
||||
return handle(task);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
class BackgroundTaskLeaseLostException extends RuntimeException {
|
||||
|
||||
BackgroundTaskLeaseLostException(Long taskId, String workerOwner) {
|
||||
super("background task lease lost: taskId=" + taskId + ", workerOwner=" + workerOwner);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface BackgroundTaskProgressReporter {
|
||||
|
||||
void report(Map<String, Object> publicStatePatch);
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface BackgroundTaskRepository extends JpaRepository<BackgroundTask, Long> {
|
||||
|
||||
Page<BackgroundTask> findByUserIdOrderByCreatedAtDesc(Long userId, Pageable pageable);
|
||||
|
||||
Optional<BackgroundTask> findByIdAndUserId(Long id, Long userId);
|
||||
|
||||
List<BackgroundTask> findByStatusOrderByCreatedAtAsc(BackgroundTaskStatus status, Pageable pageable);
|
||||
|
||||
List<BackgroundTask> findByStatusOrderByUpdatedAtAsc(BackgroundTaskStatus status);
|
||||
|
||||
@Query("""
|
||||
select task.id from BackgroundTask task
|
||||
where task.status = :status
|
||||
and (task.nextRunAt is null or task.nextRunAt <= :now)
|
||||
order by coalesce(task.nextRunAt, task.createdAt) asc, task.createdAt asc
|
||||
""")
|
||||
List<Long> findReadyTaskIdsByStatusOrder(@Param("status") BackgroundTaskStatus status,
|
||||
@Param("now") LocalDateTime now,
|
||||
Pageable pageable);
|
||||
|
||||
@Modifying
|
||||
@Query("""
|
||||
update BackgroundTask task
|
||||
set task.status = :runningStatus,
|
||||
task.errorMessage = null,
|
||||
task.nextRunAt = null,
|
||||
task.attemptCount = task.attemptCount + 1,
|
||||
task.leaseOwner = :leaseOwner,
|
||||
task.leaseExpiresAt = :leaseExpiresAt,
|
||||
task.heartbeatAt = :heartbeatAt,
|
||||
task.updatedAt = :updatedAt
|
||||
where task.id = :id
|
||||
and task.status = :queuedStatus
|
||||
""")
|
||||
int claimQueuedTask(@Param("id") Long id,
|
||||
@Param("queuedStatus") BackgroundTaskStatus queuedStatus,
|
||||
@Param("runningStatus") BackgroundTaskStatus runningStatus,
|
||||
@Param("leaseOwner") String leaseOwner,
|
||||
@Param("leaseExpiresAt") LocalDateTime leaseExpiresAt,
|
||||
@Param("heartbeatAt") LocalDateTime heartbeatAt,
|
||||
@Param("updatedAt") LocalDateTime updatedAt);
|
||||
|
||||
@Query("""
|
||||
select task.id from BackgroundTask task
|
||||
where task.status = :status
|
||||
and (task.leaseExpiresAt is null or task.leaseExpiresAt <= :now)
|
||||
order by coalesce(task.leaseExpiresAt, task.updatedAt, task.createdAt) asc
|
||||
""")
|
||||
List<Long> findExpiredRunningTaskIds(@Param("status") BackgroundTaskStatus status,
|
||||
@Param("now") LocalDateTime now,
|
||||
Pageable pageable);
|
||||
|
||||
@Modifying
|
||||
@Query("""
|
||||
update BackgroundTask task
|
||||
set task.status = :queuedStatus,
|
||||
task.errorMessage = null,
|
||||
task.finishedAt = null,
|
||||
task.nextRunAt = null,
|
||||
task.leaseOwner = null,
|
||||
task.leaseExpiresAt = null,
|
||||
task.heartbeatAt = null,
|
||||
task.updatedAt = :updatedAt
|
||||
where task.id = :id
|
||||
and task.status = :runningStatus
|
||||
and (task.leaseExpiresAt is null or task.leaseExpiresAt <= :now)
|
||||
""")
|
||||
int requeueExpiredRunningTask(@Param("id") Long id,
|
||||
@Param("runningStatus") BackgroundTaskStatus runningStatus,
|
||||
@Param("queuedStatus") BackgroundTaskStatus queuedStatus,
|
||||
@Param("now") LocalDateTime now,
|
||||
@Param("updatedAt") LocalDateTime updatedAt);
|
||||
|
||||
@Modifying
|
||||
@Query("""
|
||||
update BackgroundTask task
|
||||
set task.leaseExpiresAt = :leaseExpiresAt,
|
||||
task.heartbeatAt = :heartbeatAt,
|
||||
task.updatedAt = :updatedAt
|
||||
where task.id = :id
|
||||
and task.status = :runningStatus
|
||||
and task.leaseOwner = :leaseOwner
|
||||
""")
|
||||
int refreshRunningTaskLease(@Param("id") Long id,
|
||||
@Param("runningStatus") BackgroundTaskStatus runningStatus,
|
||||
@Param("leaseOwner") String leaseOwner,
|
||||
@Param("leaseExpiresAt") LocalDateTime leaseExpiresAt,
|
||||
@Param("heartbeatAt") LocalDateTime heartbeatAt,
|
||||
@Param("updatedAt") LocalDateTime updatedAt);
|
||||
}
|
||||
@@ -0,0 +1,683 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.api.v2.ApiV2ErrorCode;
|
||||
import com.yoyuzh.api.v2.ApiV2Exception;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class BackgroundTaskService {
|
||||
|
||||
static final String STATE_PHASE_KEY = "phase";
|
||||
static final String STATE_ATTEMPT_COUNT_KEY = "attemptCount";
|
||||
static final String STATE_MAX_ATTEMPTS_KEY = "maxAttempts";
|
||||
static final String STATE_RETRY_SCHEDULED_KEY = "retryScheduled";
|
||||
static final String STATE_NEXT_RETRY_AT_KEY = "nextRetryAt";
|
||||
static final String STATE_RETRY_DELAY_SECONDS_KEY = "retryDelaySeconds";
|
||||
static final String STATE_LAST_FAILURE_MESSAGE_KEY = "lastFailureMessage";
|
||||
static final String STATE_LAST_FAILURE_AT_KEY = "lastFailureAt";
|
||||
static final String STATE_FAILURE_CATEGORY_KEY = "failureCategory";
|
||||
static final String STATE_WORKER_OWNER_KEY = "workerOwner";
|
||||
static final String STATE_HEARTBEAT_AT_KEY = "heartbeatAt";
|
||||
static final String STATE_LEASE_EXPIRES_AT_KEY = "leaseExpiresAt";
|
||||
static final String STATE_STARTED_AT_KEY = "startedAt";
|
||||
|
||||
private static final List<String> ZIP_COMPATIBLE_EXTENSIONS = List.of(".zip", ".jar", ".war");
|
||||
private static final List<String> MEDIA_EXTENSIONS = List.of(
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg",
|
||||
".mp4", ".mov", ".mkv", ".webm", ".avi",
|
||||
".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"
|
||||
);
|
||||
private static final List<String> RETRY_TRANSIENT_STATE_KEYS = List.of(
|
||||
STATE_RETRY_SCHEDULED_KEY,
|
||||
STATE_NEXT_RETRY_AT_KEY,
|
||||
STATE_RETRY_DELAY_SECONDS_KEY,
|
||||
STATE_LAST_FAILURE_MESSAGE_KEY,
|
||||
STATE_LAST_FAILURE_AT_KEY,
|
||||
STATE_FAILURE_CATEGORY_KEY
|
||||
);
|
||||
private static final List<String> RUNNING_TRANSIENT_STATE_KEYS = List.of(
|
||||
STATE_WORKER_OWNER_KEY,
|
||||
STATE_LEASE_EXPIRES_AT_KEY
|
||||
);
|
||||
private static final int EXPIRED_RUNNING_TASK_BATCH_SIZE = 100;
|
||||
|
||||
private final BackgroundTaskRepository backgroundTaskRepository;
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask createQueuedFileTask(User user,
|
||||
BackgroundTaskType type,
|
||||
Long fileId,
|
||||
String requestedPath,
|
||||
String correlationId) {
|
||||
StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, user.getId())
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "file not found"));
|
||||
String logicalPath = buildLogicalPath(file);
|
||||
if (!logicalPath.equals(normalizeLogicalPath(requestedPath))) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task path does not match file path");
|
||||
}
|
||||
validateTaskTarget(type, file);
|
||||
|
||||
Map<String, Object> publicState = fileState(file, logicalPath);
|
||||
Map<String, Object> privateState = new LinkedHashMap<>(publicState);
|
||||
privateState.put("taskType", type.name());
|
||||
if (type == BackgroundTaskType.ARCHIVE) {
|
||||
String outputPath = file.getPath();
|
||||
String outputFilename = file.getFilename() + ".zip";
|
||||
publicState.put("outputPath", outputPath);
|
||||
publicState.put("outputFilename", outputFilename);
|
||||
privateState.put("outputPath", outputPath);
|
||||
privateState.put("outputFilename", outputFilename);
|
||||
} else if (type == BackgroundTaskType.EXTRACT) {
|
||||
String outputPath = file.getPath();
|
||||
String outputDirectoryName = deriveExtractOutputDirectoryName(file.getFilename());
|
||||
publicState.put("outputPath", outputPath);
|
||||
publicState.put("outputDirectoryName", outputDirectoryName);
|
||||
privateState.put("outputPath", outputPath);
|
||||
privateState.put("outputDirectoryName", outputDirectoryName);
|
||||
}
|
||||
return createQueuedTask(user, type, publicState, privateState, correlationId);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask createQueuedTask(User user,
|
||||
BackgroundTaskType type,
|
||||
Map<String, Object> publicState,
|
||||
Map<String, Object> privateState,
|
||||
String correlationId) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setUserId(user.getId());
|
||||
task.setType(type);
|
||||
task.setStatus(BackgroundTaskStatus.QUEUED);
|
||||
task.setAttemptCount(0);
|
||||
task.setMaxAttempts(resolveMaxAttempts(type));
|
||||
task.setNextRunAt(null);
|
||||
Map<String, Object> nextPublicState = new LinkedHashMap<>(publicState == null ? Map.of() : publicState);
|
||||
nextPublicState.put(STATE_PHASE_KEY, "queued");
|
||||
nextPublicState.putAll(retryStatePatch(task.getAttemptCount(), task.getMaxAttempts()));
|
||||
task.setPublicStateJson(toJson(nextPublicState));
|
||||
task.setPrivateStateJson(toJson(privateState));
|
||||
task.setCorrelationId(normalizeCorrelationId(correlationId));
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
public Page<BackgroundTask> listOwnedTasks(User user, Pageable pageable) {
|
||||
return backgroundTaskRepository.findByUserIdOrderByCreatedAtDesc(user.getId(), pageable);
|
||||
}
|
||||
|
||||
public BackgroundTask getOwnedTask(User user, Long id) {
|
||||
return backgroundTaskRepository.findByIdAndUserId(id, user.getId())
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask cancelOwnedTask(User user, Long id) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
|
||||
if (task.getStatus() == BackgroundTaskStatus.QUEUED || task.getStatus() == BackgroundTaskStatus.RUNNING) {
|
||||
task.setStatus(BackgroundTaskStatus.CANCELLED);
|
||||
task.setNextRunAt(null);
|
||||
clearLease(task);
|
||||
task.setPublicStateJson(mergePublicStateJson(
|
||||
task.getPublicStateJson(),
|
||||
Map.of(
|
||||
STATE_PHASE_KEY, "cancelled",
|
||||
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
|
||||
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
|
||||
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
|
||||
),
|
||||
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
|
||||
));
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(null);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
return task;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask retryOwnedTask(User user, Long id) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.getStatus() != BackgroundTaskStatus.FAILED) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "only failed tasks can be retried");
|
||||
}
|
||||
|
||||
task.setAttemptCount(0);
|
||||
task.setNextRunAt(null);
|
||||
clearLease(task);
|
||||
task.setPublicStateJson(resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts()));
|
||||
task.setStatus(BackgroundTaskStatus.QUEUED);
|
||||
task.setFinishedAt(null);
|
||||
task.setErrorMessage(null);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markRunning(User user, Long id) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
task.setStatus(BackgroundTaskStatus.RUNNING);
|
||||
task.setPublicStateJson(mergePublicStateJson(
|
||||
task.getPublicStateJson(),
|
||||
Map.of(
|
||||
STATE_PHASE_KEY, "running",
|
||||
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
|
||||
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts()
|
||||
),
|
||||
List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY)
|
||||
));
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markCompleted(User user, Long id) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
task.setStatus(BackgroundTaskStatus.COMPLETED);
|
||||
task.setNextRunAt(null);
|
||||
clearLease(task);
|
||||
task.setPublicStateJson(mergePublicStateJson(
|
||||
task.getPublicStateJson(),
|
||||
Map.of(
|
||||
STATE_PHASE_KEY, "completed",
|
||||
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
|
||||
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
|
||||
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
|
||||
),
|
||||
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
|
||||
));
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(null);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markFailed(User user, Long id, String errorMessage) {
|
||||
BackgroundTask task = getOwnedTask(user, id);
|
||||
if (task.isTerminal()) {
|
||||
return task;
|
||||
}
|
||||
task.setStatus(BackgroundTaskStatus.FAILED);
|
||||
task.setNextRunAt(null);
|
||||
clearLease(task);
|
||||
task.setPublicStateJson(mergePublicStateJson(
|
||||
task.getPublicStateJson(),
|
||||
Map.of(
|
||||
STATE_PHASE_KEY, "failed",
|
||||
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
|
||||
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
|
||||
STATE_LAST_FAILURE_MESSAGE_KEY, StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed",
|
||||
STATE_LAST_FAILURE_AT_KEY, LocalDateTime.now().toString(),
|
||||
STATE_FAILURE_CATEGORY_KEY, BackgroundTaskFailureCategory.UNKNOWN.name(),
|
||||
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
|
||||
),
|
||||
removableStateKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY), RUNNING_TRANSIENT_STATE_KEYS)
|
||||
));
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed");
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public int requeueExpiredRunningTasks() {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int recovered = 0;
|
||||
for (Long taskId : backgroundTaskRepository.findExpiredRunningTaskIds(
|
||||
BackgroundTaskStatus.RUNNING,
|
||||
now,
|
||||
PageRequest.of(0, EXPIRED_RUNNING_TASK_BATCH_SIZE)
|
||||
)) {
|
||||
int requeued = backgroundTaskRepository.requeueExpiredRunningTask(
|
||||
taskId,
|
||||
BackgroundTaskStatus.RUNNING,
|
||||
BackgroundTaskStatus.QUEUED,
|
||||
now,
|
||||
now
|
||||
);
|
||||
if (requeued != 1) {
|
||||
continue;
|
||||
}
|
||||
BackgroundTask task = backgroundTaskRepository.findById(taskId)
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
|
||||
resetTaskToQueued(task);
|
||||
backgroundTaskRepository.save(task);
|
||||
recovered += 1;
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
public List<Long> findQueuedTaskIds(int limit) {
|
||||
if (limit <= 0) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
return backgroundTaskRepository.findReadyTaskIdsByStatusOrder(
|
||||
BackgroundTaskStatus.QUEUED,
|
||||
LocalDateTime.now(),
|
||||
PageRequest.of(0, limit)
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Optional<BackgroundTask> claimQueuedTask(Long id, String workerOwner, long leaseDurationSeconds) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds));
|
||||
int claimed = backgroundTaskRepository.claimQueuedTask(
|
||||
id,
|
||||
BackgroundTaskStatus.QUEUED,
|
||||
BackgroundTaskStatus.RUNNING,
|
||||
workerOwner,
|
||||
leaseExpiresAt,
|
||||
now,
|
||||
now
|
||||
);
|
||||
if (claimed != 1) {
|
||||
return Optional.empty();
|
||||
}
|
||||
Optional<BackgroundTask> task = backgroundTaskRepository.findById(id);
|
||||
task.ifPresent(claimedTask -> {
|
||||
claimedTask.setLeaseOwner(workerOwner);
|
||||
claimedTask.setLeaseExpiresAt(leaseExpiresAt);
|
||||
claimedTask.setHeartbeatAt(now);
|
||||
claimedTask.setPublicStateJson(mergePublicStateJson(
|
||||
claimedTask.getPublicStateJson(),
|
||||
runningStatePatch(claimedTask, workerOwner, now, leaseExpiresAt, true),
|
||||
RETRY_TRANSIENT_STATE_KEYS
|
||||
));
|
||||
});
|
||||
task.ifPresent(backgroundTaskRepository::save);
|
||||
return task;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markWorkerTaskProgress(Long id,
|
||||
String workerOwner,
|
||||
Map<String, Object> publicStatePatch,
|
||||
long leaseDurationSeconds) {
|
||||
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
|
||||
BackgroundTask task = backgroundTaskRepository.findById(id)
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
|
||||
task.setLeaseOwner(workerOwner);
|
||||
task.setLeaseExpiresAt(leaseTouch.leaseExpiresAt());
|
||||
task.setHeartbeatAt(leaseTouch.now());
|
||||
Map<String, Object> nextPatch = new LinkedHashMap<>(runningStatePatch(
|
||||
task,
|
||||
workerOwner,
|
||||
leaseTouch.now(),
|
||||
leaseTouch.leaseExpiresAt(),
|
||||
false
|
||||
));
|
||||
if (publicStatePatch != null) {
|
||||
nextPatch.putAll(publicStatePatch);
|
||||
}
|
||||
task.setPublicStateJson(mergePublicStateJson(task.getPublicStateJson(), nextPatch));
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markWorkerTaskCompleted(Long id,
|
||||
String workerOwner,
|
||||
Map<String, Object> publicStatePatch,
|
||||
long leaseDurationSeconds) {
|
||||
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
|
||||
BackgroundTask task = backgroundTaskRepository.findById(id)
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
|
||||
Map<String, Object> nextPatch = new LinkedHashMap<>(publicStatePatch == null ? Map.of() : publicStatePatch);
|
||||
nextPatch.put(STATE_PHASE_KEY, "completed");
|
||||
nextPatch.put(STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount());
|
||||
nextPatch.put(STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts());
|
||||
nextPatch.put(STATE_HEARTBEAT_AT_KEY, leaseTouch.now().toString());
|
||||
task.setPublicStateJson(mergePublicStateJson(
|
||||
task.getPublicStateJson(),
|
||||
nextPatch,
|
||||
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
|
||||
));
|
||||
task.setStatus(BackgroundTaskStatus.COMPLETED);
|
||||
task.setNextRunAt(null);
|
||||
clearLease(task);
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
task.setErrorMessage(null);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BackgroundTask markWorkerTaskFailed(Long id,
|
||||
String workerOwner,
|
||||
String errorMessage,
|
||||
BackgroundTaskFailureCategory failureCategory,
|
||||
long leaseDurationSeconds) {
|
||||
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
|
||||
BackgroundTask task = backgroundTaskRepository.findById(id)
|
||||
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
|
||||
String normalizedErrorMessage = StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed";
|
||||
LocalDateTime now = leaseTouch.now();
|
||||
if (failureCategory.isRetryable() && hasRemainingAttempts(task)) {
|
||||
long retryDelaySeconds = resolveRetryDelaySeconds(task.getType(), failureCategory, task.getAttemptCount());
|
||||
LocalDateTime nextRunAt = now.plusSeconds(retryDelaySeconds);
|
||||
task.setStatus(BackgroundTaskStatus.QUEUED);
|
||||
task.setNextRunAt(nextRunAt);
|
||||
clearLease(task);
|
||||
task.setFinishedAt(null);
|
||||
task.setErrorMessage(null);
|
||||
task.setPublicStateJson(mergePublicStateJson(
|
||||
task.getPublicStateJson(),
|
||||
Map.of(
|
||||
STATE_PHASE_KEY, "queued",
|
||||
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
|
||||
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
|
||||
STATE_RETRY_SCHEDULED_KEY, true,
|
||||
STATE_NEXT_RETRY_AT_KEY, nextRunAt.toString(),
|
||||
STATE_RETRY_DELAY_SECONDS_KEY, retryDelaySeconds,
|
||||
STATE_LAST_FAILURE_MESSAGE_KEY, normalizedErrorMessage,
|
||||
STATE_LAST_FAILURE_AT_KEY, now.toString(),
|
||||
STATE_FAILURE_CATEGORY_KEY, failureCategory.name(),
|
||||
STATE_HEARTBEAT_AT_KEY, now.toString()
|
||||
),
|
||||
RUNNING_TRANSIENT_STATE_KEYS
|
||||
));
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
task.setNextRunAt(null);
|
||||
clearLease(task);
|
||||
task.setPublicStateJson(mergePublicStateJson(
|
||||
task.getPublicStateJson(),
|
||||
Map.of(
|
||||
STATE_PHASE_KEY, "failed",
|
||||
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
|
||||
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
|
||||
STATE_LAST_FAILURE_MESSAGE_KEY, normalizedErrorMessage,
|
||||
STATE_LAST_FAILURE_AT_KEY, now.toString(),
|
||||
STATE_FAILURE_CATEGORY_KEY, failureCategory.name(),
|
||||
STATE_HEARTBEAT_AT_KEY, now.toString()
|
||||
),
|
||||
removableStateKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY, STATE_RETRY_DELAY_SECONDS_KEY), RUNNING_TRANSIENT_STATE_KEYS)
|
||||
));
|
||||
task.setStatus(BackgroundTaskStatus.FAILED);
|
||||
task.setFinishedAt(now);
|
||||
task.setErrorMessage(normalizedErrorMessage);
|
||||
return backgroundTaskRepository.save(task);
|
||||
}
|
||||
|
||||
private String normalizeCorrelationId(String correlationId) {
|
||||
if (StringUtils.hasText(correlationId)) {
|
||||
return correlationId.trim();
|
||||
}
|
||||
return UUID.randomUUID().toString().replace("-", "");
|
||||
}
|
||||
|
||||
private void validateTaskTarget(BackgroundTaskType type, StoredFile file) {
|
||||
if (type == BackgroundTaskType.ARCHIVE) {
|
||||
return;
|
||||
}
|
||||
if (file.isDirectory()) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task target type is not supported");
|
||||
}
|
||||
if (type == BackgroundTaskType.EXTRACT && !isZipCompatibleArchive(file)) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "extract task only supports zip-compatible archives");
|
||||
}
|
||||
if (type == BackgroundTaskType.MEDIA_META && !isMediaLike(file)) {
|
||||
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "media metadata task only supports media files");
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> fileState(StoredFile file, String logicalPath) {
|
||||
Map<String, Object> state = new LinkedHashMap<>();
|
||||
state.put("fileId", file.getId());
|
||||
state.put("path", logicalPath);
|
||||
state.put("filename", file.getFilename());
|
||||
state.put("directory", file.isDirectory());
|
||||
state.put("contentType", file.getContentType());
|
||||
state.put("size", file.getSize());
|
||||
return state;
|
||||
}
|
||||
|
||||
private boolean isZipCompatibleArchive(StoredFile file) {
|
||||
String contentType = normalizeContentType(file.getContentType());
|
||||
if (contentType.contains("zip") || contentType.contains("java-archive")) {
|
||||
return true;
|
||||
}
|
||||
return hasExtension(file.getFilename(), ZIP_COMPATIBLE_EXTENSIONS);
|
||||
}
|
||||
|
||||
private boolean isMediaLike(StoredFile file) {
|
||||
String contentType = normalizeContentType(file.getContentType());
|
||||
if (contentType.startsWith("image/") || contentType.startsWith("video/") || contentType.startsWith("audio/")) {
|
||||
return true;
|
||||
}
|
||||
return hasExtension(file.getFilename(), MEDIA_EXTENSIONS);
|
||||
}
|
||||
|
||||
private String deriveExtractOutputDirectoryName(String filename) {
|
||||
if (!StringUtils.hasText(filename)) {
|
||||
return "extracted";
|
||||
}
|
||||
String trimmed = filename.trim();
|
||||
String lower = trimmed.toLowerCase(Locale.ROOT);
|
||||
for (String extension : ZIP_COMPATIBLE_EXTENSIONS) {
|
||||
if (lower.endsWith(extension) && trimmed.length() > extension.length()) {
|
||||
return trimmed.substring(0, trimmed.length() - extension.length());
|
||||
}
|
||||
}
|
||||
int lastDot = trimmed.lastIndexOf('.');
|
||||
if (lastDot > 0) {
|
||||
return trimmed.substring(0, lastDot);
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private boolean hasExtension(String filename, List<String> extensions) {
|
||||
if (!StringUtils.hasText(filename)) {
|
||||
return false;
|
||||
}
|
||||
String normalized = filename.toLowerCase(Locale.ROOT);
|
||||
return extensions.stream().anyMatch(normalized::endsWith);
|
||||
}
|
||||
|
||||
private String normalizeContentType(String contentType) {
|
||||
if (!StringUtils.hasText(contentType)) {
|
||||
return "";
|
||||
}
|
||||
return contentType.trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String buildLogicalPath(StoredFile file) {
|
||||
String parent = normalizeLogicalPath(file.getPath());
|
||||
if ("/".equals(parent)) {
|
||||
return "/" + file.getFilename();
|
||||
}
|
||||
return parent + "/" + file.getFilename();
|
||||
}
|
||||
|
||||
private String normalizeLogicalPath(String path) {
|
||||
if (!StringUtils.hasText(path)) {
|
||||
return "/";
|
||||
}
|
||||
String normalized = path.trim().replace('\\', '/');
|
||||
while (normalized.contains("//")) {
|
||||
normalized = normalized.replace("//", "/");
|
||||
}
|
||||
if (!normalized.startsWith("/")) {
|
||||
normalized = "/" + normalized;
|
||||
}
|
||||
while (normalized.length() > 1 && normalized.endsWith("/")) {
|
||||
normalized = normalized.substring(0, normalized.length() - 1);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String toJson(Map<String, Object> value) {
|
||||
Map<String, Object> safeValue = value == null ? new LinkedHashMap<>() : new LinkedHashMap<>(value);
|
||||
try {
|
||||
return objectMapper.writeValueAsString(safeValue);
|
||||
} catch (JsonProcessingException ex) {
|
||||
throw new IllegalStateException("Failed to serialize background task state", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> parseJsonObject(String value) {
|
||||
if (!StringUtils.hasText(value)) {
|
||||
return new LinkedHashMap<>();
|
||||
}
|
||||
|
||||
try {
|
||||
return objectMapper.readValue(value, new TypeReference<LinkedHashMap<String, Object>>() {
|
||||
});
|
||||
} catch (JsonProcessingException ex) {
|
||||
throw new IllegalStateException("Failed to parse background task state", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private String mergePublicStateJson(String currentValue, Map<String, Object> patch) {
|
||||
return mergePublicStateJson(currentValue, patch, List.of());
|
||||
}
|
||||
|
||||
private String mergePublicStateJson(String currentValue, Map<String, Object> patch, List<String> keysToRemove) {
|
||||
Map<String, Object> nextPublicState = parseJsonObject(currentValue);
|
||||
if (keysToRemove != null) {
|
||||
keysToRemove.forEach(nextPublicState::remove);
|
||||
}
|
||||
if (patch != null) {
|
||||
nextPublicState.putAll(patch);
|
||||
}
|
||||
return toJson(nextPublicState);
|
||||
}
|
||||
|
||||
private String resetPublicStateForRetry(String privateStateJson, int attemptCount, int maxAttempts) {
|
||||
Map<String, Object> nextPublicState = parseJsonObject(privateStateJson);
|
||||
nextPublicState.remove("taskType");
|
||||
nextPublicState.put(STATE_PHASE_KEY, "queued");
|
||||
nextPublicState.putAll(retryStatePatch(attemptCount, maxAttempts));
|
||||
return toJson(nextPublicState);
|
||||
}
|
||||
|
||||
private void resetTaskToQueued(BackgroundTask task) {
|
||||
task.setNextRunAt(null);
|
||||
clearLease(task);
|
||||
task.setPublicStateJson(resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts()));
|
||||
task.setStatus(BackgroundTaskStatus.QUEUED);
|
||||
task.setFinishedAt(null);
|
||||
task.setErrorMessage(null);
|
||||
}
|
||||
|
||||
private int resolveMaxAttempts(BackgroundTaskType type) {
|
||||
return switch (type) {
|
||||
case ARCHIVE -> 4;
|
||||
case EXTRACT -> 3;
|
||||
case MEDIA_META -> 2;
|
||||
default -> 1;
|
||||
};
|
||||
}
|
||||
|
||||
private Map<String, Object> retryStatePatch(int attemptCount, int maxAttempts) {
|
||||
Map<String, Object> patch = new LinkedHashMap<>();
|
||||
patch.put(STATE_ATTEMPT_COUNT_KEY, attemptCount);
|
||||
patch.put(STATE_MAX_ATTEMPTS_KEY, maxAttempts);
|
||||
return patch;
|
||||
}
|
||||
|
||||
private boolean hasRemainingAttempts(BackgroundTask task) {
|
||||
return task.getAttemptCount() != null
|
||||
&& task.getMaxAttempts() != null
|
||||
&& task.getAttemptCount() < task.getMaxAttempts();
|
||||
}
|
||||
|
||||
private long resolveRetryDelaySeconds(BackgroundTaskType type,
|
||||
BackgroundTaskFailureCategory failureCategory,
|
||||
Integer attemptCount) {
|
||||
int safeAttemptCount = attemptCount == null ? 1 : Math.max(1, attemptCount);
|
||||
long baseDelaySeconds = switch (type) {
|
||||
case ARCHIVE -> 30L;
|
||||
case EXTRACT -> 45L;
|
||||
case MEDIA_META -> 15L;
|
||||
default -> 30L;
|
||||
};
|
||||
if (failureCategory == BackgroundTaskFailureCategory.RATE_LIMITED) {
|
||||
baseDelaySeconds *= 4L;
|
||||
} else if (failureCategory == BackgroundTaskFailureCategory.UNKNOWN) {
|
||||
baseDelaySeconds *= 2L;
|
||||
}
|
||||
long delay = baseDelaySeconds * (1L << Math.min(safeAttemptCount - 1, 2));
|
||||
return Math.min(delay, baseDelaySeconds * 4L);
|
||||
}
|
||||
|
||||
private LeaseTouch refreshLease(Long id, String workerOwner, long leaseDurationSeconds) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds));
|
||||
int refreshed = backgroundTaskRepository.refreshRunningTaskLease(
|
||||
id,
|
||||
BackgroundTaskStatus.RUNNING,
|
||||
workerOwner,
|
||||
leaseExpiresAt,
|
||||
now,
|
||||
now
|
||||
);
|
||||
if (refreshed != 1) {
|
||||
throw new BackgroundTaskLeaseLostException(id, workerOwner);
|
||||
}
|
||||
return new LeaseTouch(now, leaseExpiresAt);
|
||||
}
|
||||
|
||||
private Map<String, Object> runningStatePatch(BackgroundTask task,
|
||||
String workerOwner,
|
||||
LocalDateTime heartbeatAt,
|
||||
LocalDateTime leaseExpiresAt,
|
||||
boolean includeStartedAt) {
|
||||
Map<String, Object> patch = new LinkedHashMap<>();
|
||||
patch.put(STATE_PHASE_KEY, "running");
|
||||
patch.put(STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount());
|
||||
patch.put(STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts());
|
||||
patch.put(STATE_WORKER_OWNER_KEY, workerOwner);
|
||||
patch.put(STATE_HEARTBEAT_AT_KEY, heartbeatAt.toString());
|
||||
patch.put(STATE_LEASE_EXPIRES_AT_KEY, leaseExpiresAt.toString());
|
||||
if (includeStartedAt) {
|
||||
patch.put(STATE_STARTED_AT_KEY, heartbeatAt.toString());
|
||||
}
|
||||
return patch;
|
||||
}
|
||||
|
||||
private List<String> removableStateKeys(List<String> primary, List<String> secondary) {
|
||||
List<String> keys = new java.util.ArrayList<>(primary);
|
||||
keys.addAll(secondary);
|
||||
return keys;
|
||||
}
|
||||
|
||||
private void clearLease(BackgroundTask task) {
|
||||
task.setLeaseOwner(null);
|
||||
task.setLeaseExpiresAt(null);
|
||||
task.setHeartbeatAt(null);
|
||||
}
|
||||
|
||||
private record LeaseTouch(LocalDateTime now, LocalDateTime leaseExpiresAt) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class BackgroundTaskStartupRecovery {
|
||||
|
||||
private final BackgroundTaskService backgroundTaskService;
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void recoverOnStartup() {
|
||||
int recovered = backgroundTaskService.requeueExpiredRunningTasks();
|
||||
if (recovered > 0) {
|
||||
log.warn("Recovered {} expired RUNNING background task leases back to QUEUED on startup", recovered);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
public enum BackgroundTaskStatus {
|
||||
QUEUED,
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
public enum BackgroundTaskType {
|
||||
ARCHIVE,
|
||||
@@ -0,0 +1,167 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.net.ConnectException;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
@Component
|
||||
public class BackgroundTaskWorker {
|
||||
|
||||
private static final int DEFAULT_BATCH_SIZE = 5;
|
||||
private static final long DEFAULT_LEASE_DURATION_SECONDS = 120L;
|
||||
|
||||
private final BackgroundTaskService backgroundTaskService;
|
||||
private final List<BackgroundTaskHandler> handlers;
|
||||
private final String workerOwner;
|
||||
|
||||
public BackgroundTaskWorker(BackgroundTaskService backgroundTaskService,
|
||||
List<BackgroundTaskHandler> handlers) {
|
||||
this.backgroundTaskService = backgroundTaskService;
|
||||
this.handlers = List.copyOf(handlers);
|
||||
this.workerOwner = UUID.randomUUID().toString().replace("-", "");
|
||||
}
|
||||
|
||||
@Scheduled(
|
||||
fixedDelayString = "${app.background-tasks.worker.fixed-delay-ms:30000}",
|
||||
initialDelayString = "${app.background-tasks.worker.initial-delay-ms:30000}"
|
||||
)
|
||||
public void runScheduledBatch() {
|
||||
processQueuedTasks(DEFAULT_BATCH_SIZE);
|
||||
}
|
||||
|
||||
public int processQueuedTasks(int maxTasks) {
|
||||
backgroundTaskService.requeueExpiredRunningTasks();
|
||||
int processedCount = 0;
|
||||
for (Long taskId : backgroundTaskService.findQueuedTaskIds(maxTasks)) {
|
||||
var claimedTask = backgroundTaskService.claimQueuedTask(taskId, workerOwner, DEFAULT_LEASE_DURATION_SECONDS);
|
||||
if (claimedTask.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
execute(claimedTask.get());
|
||||
processedCount += 1;
|
||||
}
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
private void execute(BackgroundTask task) {
|
||||
try {
|
||||
backgroundTaskService.markWorkerTaskProgress(
|
||||
task.getId(),
|
||||
workerOwner,
|
||||
Map.of(BackgroundTaskService.STATE_PHASE_KEY, resolveRunningPhase(task.getType())),
|
||||
DEFAULT_LEASE_DURATION_SECONDS
|
||||
);
|
||||
BackgroundTaskHandler handler = findHandler(task);
|
||||
BackgroundTaskHandlerResult result = handler.handle(task, publicStatePatch ->
|
||||
backgroundTaskService.markWorkerTaskProgress(
|
||||
task.getId(),
|
||||
workerOwner,
|
||||
publicStatePatch,
|
||||
DEFAULT_LEASE_DURATION_SECONDS
|
||||
));
|
||||
backgroundTaskService.markWorkerTaskCompleted(
|
||||
task.getId(),
|
||||
workerOwner,
|
||||
result.publicStatePatch(),
|
||||
DEFAULT_LEASE_DURATION_SECONDS
|
||||
);
|
||||
} catch (BackgroundTaskLeaseLostException ignored) {
|
||||
// Another worker reclaimed the task after this worker stopped heartbeating.
|
||||
} catch (Exception ex) {
|
||||
String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage();
|
||||
try {
|
||||
backgroundTaskService.markWorkerTaskFailed(
|
||||
task.getId(),
|
||||
workerOwner,
|
||||
message,
|
||||
classifyFailure(ex),
|
||||
DEFAULT_LEASE_DURATION_SECONDS
|
||||
);
|
||||
} catch (BackgroundTaskLeaseLostException ignored) {
|
||||
// Lease already moved to another worker; keep current worker from overwriting state.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveRunningPhase(BackgroundTaskType type) {
|
||||
return switch (type) {
|
||||
case ARCHIVE -> "archiving";
|
||||
case EXTRACT -> "extracting";
|
||||
case MEDIA_META -> "extracting-metadata";
|
||||
default -> "running";
|
||||
};
|
||||
}
|
||||
|
||||
private BackgroundTaskHandler findHandler(BackgroundTask task) {
|
||||
return handlers.stream()
|
||||
.filter(handler -> handler.supports(task.getType()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IllegalStateException("No background task handler for " + task.getType()));
|
||||
}
|
||||
|
||||
private BackgroundTaskFailureCategory classifyFailure(Throwable throwable) {
|
||||
if (containsRateLimitSignal(throwable)) {
|
||||
return BackgroundTaskFailureCategory.RATE_LIMITED;
|
||||
}
|
||||
Throwable current = throwable;
|
||||
while (current != null) {
|
||||
if (current instanceof BusinessException || current instanceof IllegalArgumentException) {
|
||||
return BackgroundTaskFailureCategory.UNSUPPORTED_INPUT;
|
||||
}
|
||||
if (current instanceof IllegalStateException) {
|
||||
return BackgroundTaskFailureCategory.DATA_STATE;
|
||||
}
|
||||
if (current instanceof SocketTimeoutException
|
||||
|| current instanceof TimeoutException
|
||||
|| current instanceof ConnectException
|
||||
|| current instanceof IOException
|
||||
|| current instanceof UncheckedIOException) {
|
||||
return BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE;
|
||||
}
|
||||
current = current.getCause();
|
||||
}
|
||||
if (containsTransientInfrastructureSignal(throwable)) {
|
||||
return BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE;
|
||||
}
|
||||
return BackgroundTaskFailureCategory.UNKNOWN;
|
||||
}
|
||||
|
||||
private boolean containsRateLimitSignal(Throwable throwable) {
|
||||
String message = collectMessages(throwable);
|
||||
return message.contains("429")
|
||||
|| message.contains("too many requests")
|
||||
|| message.contains("rate limit")
|
||||
|| message.contains("throttle");
|
||||
}
|
||||
|
||||
private boolean containsTransientInfrastructureSignal(Throwable throwable) {
|
||||
String message = collectMessages(throwable);
|
||||
return message.contains("timeout")
|
||||
|| message.contains("temporarily unavailable")
|
||||
|| message.contains("connection reset")
|
||||
|| message.contains("broken pipe")
|
||||
|| message.contains("connection refused");
|
||||
}
|
||||
|
||||
private String collectMessages(Throwable throwable) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
Throwable current = throwable;
|
||||
while (current != null) {
|
||||
if (current.getMessage() != null) {
|
||||
builder.append(current.getMessage().toLowerCase()).append(' ');
|
||||
}
|
||||
current = current.getCause();
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import jakarta.transaction.Transactional;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.net.URLConnection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
@Component
|
||||
@Transactional
|
||||
public class ExtractBackgroundTaskHandler implements BackgroundTaskHandler {
|
||||
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final UserRepository userRepository;
|
||||
private final FileService fileService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public ExtractBackgroundTaskHandler(StoredFileRepository storedFileRepository,
|
||||
UserRepository userRepository,
|
||||
FileService fileService,
|
||||
ObjectMapper objectMapper) {
|
||||
this.storedFileRepository = storedFileRepository;
|
||||
this.userRepository = userRepository;
|
||||
this.fileService = fileService;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supports(BackgroundTaskType type) {
|
||||
return type == BackgroundTaskType.EXTRACT;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BackgroundTaskHandlerResult handle(BackgroundTask task) {
|
||||
return handle(task, publicStatePatch -> {
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
|
||||
Map<String, Object> state = parseState(task.getPrivateStateJson(), task.getPublicStateJson());
|
||||
Long fileId = extractLong(state.get("fileId"));
|
||||
String outputPath = extractText(state.get("outputPath"));
|
||||
String outputDirectoryName = extractText(state.get("outputDirectoryName"));
|
||||
if (fileId == null) {
|
||||
throw new IllegalStateException("extract task missing fileId");
|
||||
}
|
||||
if (!StringUtils.hasText(outputPath) || !StringUtils.hasText(outputDirectoryName)) {
|
||||
throw new IllegalStateException("extract task missing output target");
|
||||
}
|
||||
|
||||
StoredFile archive = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId())
|
||||
.orElseThrow(() -> new IllegalStateException("extract task file not found"));
|
||||
if (archive.isDirectory()) {
|
||||
throw new IllegalStateException("extract task only supports files");
|
||||
}
|
||||
User user = userRepository.findById(task.getUserId())
|
||||
.orElseThrow(() -> new IllegalStateException("extract task user not found"));
|
||||
|
||||
ExtractPlan plan = parseArchivePlan(archive, outputPath, outputDirectoryName);
|
||||
progressReporter.report(progressPatch(0, plan.files().size(), 0, plan.directories().size()));
|
||||
executePlan(user, plan, progressReporter);
|
||||
|
||||
Map<String, Object> publicStatePatch = new LinkedHashMap<>();
|
||||
publicStatePatch.put("worker", "extract");
|
||||
publicStatePatch.put("extractedPath", plan.extractedPath());
|
||||
publicStatePatch.put("extractedFileCount", plan.files().size());
|
||||
publicStatePatch.put("extractedDirectoryCount", plan.directories().size());
|
||||
publicStatePatch.putAll(progressPatch(
|
||||
plan.files().size(),
|
||||
plan.files().size(),
|
||||
plan.directories().size(),
|
||||
plan.directories().size()
|
||||
));
|
||||
return new BackgroundTaskHandlerResult(publicStatePatch);
|
||||
}
|
||||
|
||||
private void executePlan(User user, ExtractPlan plan, BackgroundTaskProgressReporter progressReporter) {
|
||||
fileService.importExternalFilesAtomically(
|
||||
user,
|
||||
plan.directories(),
|
||||
plan.files().stream()
|
||||
.map(file -> new FileService.ExternalFileImport(
|
||||
file.parentPath(),
|
||||
file.filename(),
|
||||
file.contentType(),
|
||||
file.content()
|
||||
))
|
||||
.toList(),
|
||||
progress -> progressReporter.report(progressPatch(
|
||||
progress.processedFileCount(),
|
||||
progress.totalFileCount(),
|
||||
progress.processedDirectoryCount(),
|
||||
progress.totalDirectoryCount()
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
private Map<String, Object> progressPatch(int processedFileCount,
|
||||
int totalFileCount,
|
||||
int processedDirectoryCount,
|
||||
int totalDirectoryCount) {
|
||||
Map<String, Object> patch = new LinkedHashMap<>();
|
||||
patch.put("processedFileCount", processedFileCount);
|
||||
patch.put("totalFileCount", totalFileCount);
|
||||
patch.put("processedDirectoryCount", processedDirectoryCount);
|
||||
patch.put("totalDirectoryCount", totalDirectoryCount);
|
||||
patch.put("progressPercent", calculateProgressPercent(
|
||||
processedFileCount,
|
||||
totalFileCount,
|
||||
processedDirectoryCount,
|
||||
totalDirectoryCount
|
||||
));
|
||||
return patch;
|
||||
}
|
||||
|
||||
private int calculateProgressPercent(int processedFileCount,
|
||||
int totalFileCount,
|
||||
int processedDirectoryCount,
|
||||
int totalDirectoryCount) {
|
||||
int total = Math.max(0, totalFileCount) + Math.max(0, totalDirectoryCount);
|
||||
int processed = Math.max(0, processedFileCount) + Math.max(0, processedDirectoryCount);
|
||||
if (total <= 0) {
|
||||
return 100;
|
||||
}
|
||||
return Math.min(100, (int) Math.floor((processed * 100.0d) / total));
|
||||
}
|
||||
|
||||
private ExtractPlan parseArchivePlan(StoredFile archive, String outputPath, String outputDirectoryName) {
|
||||
FileService.ZipCompatibleArchive zipArchive;
|
||||
try {
|
||||
zipArchive = fileService.readZipCompatibleArchive(archive);
|
||||
} catch (BusinessException ex) {
|
||||
throw new IllegalStateException("extract task only supports zip-compatible archives", ex);
|
||||
}
|
||||
|
||||
List<ZipItem> items = zipArchive.entries().stream()
|
||||
.map(entry -> toZipItem(entry, zipArchive.commonRootDirectoryName()))
|
||||
.filter(item -> StringUtils.hasText(item.path()))
|
||||
.toList();
|
||||
if (items.isEmpty()) {
|
||||
throw new IllegalStateException("extract task archive is empty");
|
||||
}
|
||||
|
||||
String normalizedOutputPath = normalizeDirectoryPath(outputPath);
|
||||
if (shouldExtractSingleFileToParent(items, outputDirectoryName)) {
|
||||
ZipItem fileItem = items.get(0);
|
||||
return new ExtractPlan(
|
||||
List.of(),
|
||||
List.of(new ExtractedFile(
|
||||
normalizedOutputPath,
|
||||
outputDirectoryName,
|
||||
fileItem.contentType(),
|
||||
fileItem.content()
|
||||
)),
|
||||
normalizedOutputPath
|
||||
);
|
||||
}
|
||||
|
||||
String rootPath = joinPath(normalizedOutputPath, outputDirectoryName);
|
||||
LinkedHashSet<String> directories = new LinkedHashSet<>();
|
||||
directories.add(rootPath);
|
||||
List<ExtractedFile> files = new ArrayList<>();
|
||||
for (ZipItem item : items) {
|
||||
if (item.directory()) {
|
||||
directories.add(joinPath(rootPath, trimTrailingSlash(item.path())));
|
||||
continue;
|
||||
}
|
||||
String relativeParent = extractParentPath(item.path());
|
||||
String targetParent = StringUtils.hasText(relativeParent) ? joinPath(rootPath, relativeParent) : rootPath;
|
||||
collectParentDirectories(directories, rootPath, relativeParent);
|
||||
files.add(new ExtractedFile(
|
||||
targetParent,
|
||||
extractLeafName(item.path()),
|
||||
item.contentType(),
|
||||
item.content()
|
||||
));
|
||||
}
|
||||
|
||||
return new ExtractPlan(List.copyOf(directories), List.copyOf(files), rootPath);
|
||||
}
|
||||
|
||||
private void collectParentDirectories(LinkedHashSet<String> directories, String rootPath, String relativeParent) {
|
||||
if (!StringUtils.hasText(relativeParent)) {
|
||||
return;
|
||||
}
|
||||
String current = "";
|
||||
for (String segment : relativeParent.split("/")) {
|
||||
current = StringUtils.hasText(current) ? current + "/" + segment : segment;
|
||||
directories.add(joinPath(rootPath, current));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean shouldExtractSingleFileToParent(List<ZipItem> items, String outputDirectoryName) {
|
||||
if (items.size() != 1) {
|
||||
return false;
|
||||
}
|
||||
ZipItem item = items.get(0);
|
||||
return !item.directory()
|
||||
&& !item.path().contains("/")
|
||||
&& outputDirectoryName.equals(item.path());
|
||||
}
|
||||
|
||||
private ZipItem toZipItem(FileService.ZipCompatibleArchiveEntry entry, String commonRootDirectoryName) {
|
||||
String path = stripCommonRootDirectory(entry.relativePath(), commonRootDirectoryName);
|
||||
return new ZipItem(path, entry.directory(), entry.content(), guessContentType(path));
|
||||
}
|
||||
|
||||
private String stripCommonRootDirectory(String relativePath, String commonRootDirectoryName) {
|
||||
if (!StringUtils.hasText(relativePath) || !StringUtils.hasText(commonRootDirectoryName)) {
|
||||
return relativePath;
|
||||
}
|
||||
String prefix = commonRootDirectoryName + "/";
|
||||
if (relativePath.equals(commonRootDirectoryName)) {
|
||||
return "";
|
||||
}
|
||||
if (relativePath.startsWith(prefix)) {
|
||||
return relativePath.substring(prefix.length());
|
||||
}
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
private Map<String, Object> parseState(String privateStateJson, String publicStateJson) {
|
||||
Map<String, Object> state = new LinkedHashMap<>(parseJsonObject(publicStateJson));
|
||||
state.putAll(parseJsonObject(privateStateJson));
|
||||
return state;
|
||||
}
|
||||
|
||||
private Map<String, Object> parseJsonObject(String json) {
|
||||
if (!StringUtils.hasText(json)) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
|
||||
});
|
||||
} catch (JsonProcessingException ex) {
|
||||
throw new IllegalStateException("extract task state is invalid", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private Long extractLong(Object value) {
|
||||
if (value instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
if (value instanceof String text && StringUtils.hasText(text)) {
|
||||
return Long.parseLong(text.trim());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractText(Object value) {
|
||||
if (value instanceof String text && StringUtils.hasText(text)) {
|
||||
return text.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String normalizeDirectoryPath(String path) {
|
||||
if (!StringUtils.hasText(path)) {
|
||||
return "/";
|
||||
}
|
||||
String normalized = path.trim().replace('\\', '/');
|
||||
while (normalized.contains("//")) {
|
||||
normalized = normalized.replace("//", "/");
|
||||
}
|
||||
if (!normalized.startsWith("/")) {
|
||||
normalized = "/" + normalized;
|
||||
}
|
||||
while (normalized.length() > 1 && normalized.endsWith("/")) {
|
||||
normalized = normalized.substring(0, normalized.length() - 1);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String joinPath(String parent, String leaf) {
|
||||
String normalizedParent = normalizeDirectoryPath(parent);
|
||||
String normalizedLeaf = trimSlashes(leaf);
|
||||
if (!StringUtils.hasText(normalizedLeaf)) {
|
||||
return normalizedParent;
|
||||
}
|
||||
if ("/".equals(normalizedParent)) {
|
||||
return "/" + normalizedLeaf;
|
||||
}
|
||||
return normalizedParent + "/" + normalizedLeaf;
|
||||
}
|
||||
|
||||
private String trimSlashes(String value) {
|
||||
String normalized = value == null ? "" : value.trim().replace('\\', '/');
|
||||
while (normalized.startsWith("/")) {
|
||||
normalized = normalized.substring(1);
|
||||
}
|
||||
while (normalized.endsWith("/")) {
|
||||
normalized = normalized.substring(0, normalized.length() - 1);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String trimTrailingSlash(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
String normalized = value;
|
||||
while (normalized.endsWith("/")) {
|
||||
normalized = normalized.substring(0, normalized.length() - 1);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String extractParentPath(String path) {
|
||||
int separator = path.lastIndexOf('/');
|
||||
if (separator < 0) {
|
||||
return "";
|
||||
}
|
||||
return path.substring(0, separator);
|
||||
}
|
||||
|
||||
private String extractLeafName(String path) {
|
||||
int separator = path.lastIndexOf('/');
|
||||
if (separator < 0) {
|
||||
return path;
|
||||
}
|
||||
return path.substring(separator + 1);
|
||||
}
|
||||
|
||||
private String guessContentType(String entryPath) {
|
||||
String contentType = URLConnection.guessContentTypeFromName(entryPath);
|
||||
if (StringUtils.hasText(contentType)) {
|
||||
return contentType;
|
||||
}
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
private record ZipItem(String path, boolean directory, byte[] content, String contentType) {
|
||||
}
|
||||
|
||||
private record ExtractedFile(String parentPath, String filename, String contentType, byte[] content) {
|
||||
}
|
||||
|
||||
private record ExtractPlan(List<String> directories, List<ExtractedFile> files, String extractedPath) {
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,13 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.search.FileMetadata;
|
||||
import com.yoyuzh.files.search.FileMetadataRepository;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import jakarta.transaction.Transactional;
|
||||
import org.springframework.stereotype.Component;
|
||||
@@ -48,6 +53,12 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
|
||||
|
||||
@Override
|
||||
public BackgroundTaskHandlerResult handle(BackgroundTask task) {
|
||||
return handle(task, publicStatePatch -> {
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
|
||||
Long fileId = readFileId(task);
|
||||
StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId())
|
||||
.orElseThrow(() -> new IllegalStateException("media metadata task file not found"));
|
||||
@@ -62,6 +73,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
|
||||
|
||||
String contentType = firstText(file.getContentType(), blob.getContentType());
|
||||
long size = firstLong(file.getSize(), blob.getSize());
|
||||
progressReporter.report(Map.of("metadataStage", "loading-content"));
|
||||
byte[] content = Optional.ofNullable(fileContentStorage.readBlob(blob.getObjectKey()))
|
||||
.orElseThrow(() -> new IllegalStateException("media metadata task requires blob content"));
|
||||
|
||||
@@ -75,6 +87,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
|
||||
upsertMetadata(file, MEDIA_SIZE, String.valueOf(size));
|
||||
|
||||
try {
|
||||
progressReporter.report(Map.of("metadataStage", "reading-image"));
|
||||
BufferedImage image = ImageIO.read(new ByteArrayInputStream(content));
|
||||
if (image != null) {
|
||||
upsertMetadata(file, MEDIA_WIDTH, String.valueOf(image.getWidth()));
|
||||
@@ -86,6 +99,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
|
||||
throw new IllegalStateException("media metadata task failed to read image dimensions", ex);
|
||||
}
|
||||
|
||||
publicStatePatch.put("metadataStage", "completed");
|
||||
return new BackgroundTaskHandlerResult(publicStatePatch);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -9,10 +9,7 @@ import java.util.Set;
|
||||
@Component
|
||||
public class NoopBackgroundTaskHandler implements BackgroundTaskHandler {
|
||||
|
||||
private static final Set<BackgroundTaskType> SUPPORTED_TYPES = Set.of(
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
BackgroundTaskType.EXTRACT
|
||||
);
|
||||
private static final Set<BackgroundTaskType> SUPPORTED_TYPES = Set.of();
|
||||
|
||||
@Override
|
||||
public boolean supports(BackgroundTaskType type) {
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import jakarta.persistence.Column;
|
||||
@@ -55,6 +55,9 @@ public class UploadSession {
|
||||
@Column(name = "object_key", nullable = false, length = 512)
|
||||
private String objectKey;
|
||||
|
||||
@Column(name = "multipart_upload_id", length = 255)
|
||||
private String multipartUploadId;
|
||||
|
||||
@Column(name = "storage_policy_id")
|
||||
private Long storagePolicyId;
|
||||
|
||||
@@ -163,6 +166,14 @@ public class UploadSession {
|
||||
this.objectKey = objectKey;
|
||||
}
|
||||
|
||||
public String getMultipartUploadId() {
|
||||
return multipartUploadId;
|
||||
}
|
||||
|
||||
public void setMultipartUploadId(String multipartUploadId) {
|
||||
this.multipartUploadId = multipartUploadId;
|
||||
}
|
||||
|
||||
public Long getStoragePolicyId() {
|
||||
return storagePolicyId;
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
public record UploadSessionCreateCommand(
|
||||
String path,
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
public record UploadSessionPartCommand(
|
||||
String etag,
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
@@ -6,7 +6,13 @@ import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.ErrorCode;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicy;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import com.yoyuzh.files.storage.MultipartCompletedPart;
|
||||
import com.yoyuzh.files.storage.PreparedUpload;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -82,7 +88,8 @@ public class UploadSessionService {
|
||||
session.setContentType(command.contentType());
|
||||
session.setSize(command.size());
|
||||
session.setObjectKey(createBlobObjectKey());
|
||||
session.setStoragePolicyId(storagePolicyService.ensureDefaultPolicy().getId());
|
||||
StoragePolicy policy = storagePolicyService.ensureDefaultPolicy();
|
||||
session.setStoragePolicyId(policy.getId());
|
||||
session.setChunkSize(DEFAULT_CHUNK_SIZE);
|
||||
session.setChunkCount(calculateChunkCount(command.size(), DEFAULT_CHUNK_SIZE));
|
||||
session.setUploadedPartsJson("[]");
|
||||
@@ -91,6 +98,9 @@ public class UploadSessionService {
|
||||
session.setCreatedAt(now);
|
||||
session.setUpdatedAt(now);
|
||||
session.setExpiresAt(now.plusHours(SESSION_TTL_HOURS));
|
||||
if (storagePolicyService.readCapabilities(policy).multipartUpload()) {
|
||||
session.setMultipartUploadId(fileContentStorage.createMultipartUpload(session.getObjectKey(), session.getContentType()));
|
||||
}
|
||||
return uploadSessionRepository.save(session);
|
||||
}
|
||||
|
||||
@@ -111,6 +121,26 @@ public class UploadSessionService {
|
||||
return uploadSessionRepository.save(session);
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public PreparedUpload prepareOwnedPartUpload(User user, String sessionId, int partIndex) {
|
||||
UploadSession session = getOwnedSession(user, sessionId);
|
||||
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone());
|
||||
ensureSessionCanReceivePart(session, now);
|
||||
if (!StringUtils.hasText(session.getMultipartUploadId())) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话未启用 multipart");
|
||||
}
|
||||
if (partIndex < 0 || partIndex >= session.getChunkCount()) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法");
|
||||
}
|
||||
return fileContentStorage.prepareMultipartPartUpload(
|
||||
session.getObjectKey(),
|
||||
session.getMultipartUploadId(),
|
||||
partIndex + 1,
|
||||
session.getContentType(),
|
||||
resolveChunkSize(session, partIndex)
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public UploadSession recordUploadedPart(User user,
|
||||
String sessionId,
|
||||
@@ -164,6 +194,13 @@ public class UploadSessionService {
|
||||
uploadSessionRepository.save(session);
|
||||
|
||||
try {
|
||||
if (StringUtils.hasText(session.getMultipartUploadId())) {
|
||||
fileContentStorage.completeMultipartUpload(
|
||||
session.getObjectKey(),
|
||||
session.getMultipartUploadId(),
|
||||
toCompletedParts(session)
|
||||
);
|
||||
}
|
||||
fileService.completeUpload(user, new CompleteUploadRequest(
|
||||
session.getTargetPath(),
|
||||
session.getFilename(),
|
||||
@@ -192,7 +229,11 @@ public class UploadSessionService {
|
||||
);
|
||||
for (UploadSession session : expiredSessions) {
|
||||
try {
|
||||
fileContentStorage.deleteBlob(session.getObjectKey());
|
||||
if (StringUtils.hasText(session.getMultipartUploadId())) {
|
||||
fileContentStorage.abortMultipartUpload(session.getObjectKey(), session.getMultipartUploadId());
|
||||
} else {
|
||||
fileContentStorage.deleteBlob(session.getObjectKey());
|
||||
}
|
||||
} catch (RuntimeException ignored) {
|
||||
// Expiration is authoritative in the database even if remote object cleanup fails.
|
||||
}
|
||||
@@ -253,6 +294,41 @@ public class UploadSessionService {
|
||||
}
|
||||
}
|
||||
|
||||
private List<MultipartCompletedPart> toCompletedParts(UploadSession session) {
|
||||
List<UploadedPart> uploadedParts = readUploadedParts(session).stream()
|
||||
.sorted(Comparator.comparingInt(UploadedPart::partIndex))
|
||||
.toList();
|
||||
if (uploadedParts.size() != session.getChunkCount()) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片不完整");
|
||||
}
|
||||
for (int expectedIndex = 0; expectedIndex < session.getChunkCount(); expectedIndex++) {
|
||||
UploadedPart part = uploadedParts.get(expectedIndex);
|
||||
if (part.partIndex() != expectedIndex) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片不完整");
|
||||
}
|
||||
if (!StringUtils.hasText(part.etag())) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片标识缺失");
|
||||
}
|
||||
if (part.size() <= 0 || part.size() > resolveChunkSize(session, expectedIndex)) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片大小不合法");
|
||||
}
|
||||
}
|
||||
return uploadedParts.stream()
|
||||
.map(part -> new MultipartCompletedPart(part.partIndex() + 1, part.etag()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private long resolveChunkSize(UploadSession session, int partIndex) {
|
||||
if (partIndex < 0 || partIndex >= session.getChunkCount()) {
|
||||
throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法");
|
||||
}
|
||||
if (partIndex < session.getChunkCount() - 1) {
|
||||
return session.getChunkSize();
|
||||
}
|
||||
long remaining = session.getSize() - session.getChunkSize() * (session.getChunkCount() - 1L);
|
||||
return remaining > 0 ? remaining : session.getChunkSize();
|
||||
}
|
||||
|
||||
private record UploadedPart(int partIndex, String etag, long size, String uploadedAt) {
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
public enum UploadSessionStatus {
|
||||
CREATED,
|
||||
@@ -5,8 +5,8 @@ import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.ApiResponse;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.ErrorCode;
|
||||
import com.yoyuzh.files.FileMetadataResponse;
|
||||
import com.yoyuzh.files.ImportSharedFileRequest;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.share.ImportSharedFileRequest;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -5,8 +5,8 @@ import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.ErrorCode;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.FileMetadataResponse;
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
|
||||
@@ -4,10 +4,10 @@ import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.admin.AdminMetricsStateRepository;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.FileBlob;
|
||||
import com.yoyuzh.files.FileBlobRepository;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.transfer.OfflineTransferSessionRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -8,12 +8,12 @@ import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.auth.UserRole;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.FileBlobRepository;
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.StoragePolicyRepository;
|
||||
import com.yoyuzh.files.StoragePolicyService;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicyRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.transfer.OfflineTransferSessionRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -4,9 +4,9 @@ import com.yoyuzh.api.v2.ApiV2ExceptionHandler;
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.FileMetadataResponse;
|
||||
import com.yoyuzh.files.FileSearchQuery;
|
||||
import com.yoyuzh.files.FileSearchService;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.search.FileSearchQuery;
|
||||
import com.yoyuzh.files.search.FileSearchService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.MethodParameter;
|
||||
|
||||
@@ -2,9 +2,9 @@ package com.yoyuzh.api.v2.files;
|
||||
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.UploadSession;
|
||||
import com.yoyuzh.files.UploadSessionService;
|
||||
import com.yoyuzh.files.UploadSessionStatus;
|
||||
import com.yoyuzh.files.upload.UploadSession;
|
||||
import com.yoyuzh.files.upload.UploadSessionService;
|
||||
import com.yoyuzh.files.upload.UploadSessionStatus;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.MediaType;
|
||||
@@ -19,6 +19,7 @@ import org.springframework.web.method.support.HandlerMethodArgumentResolver;
|
||||
import org.springframework.web.method.support.ModelAndViewContainer;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
@@ -69,6 +70,7 @@ class UploadSessionV2ControllerTest {
|
||||
.andExpect(jsonPath("$.data.sessionId").value("session-1"))
|
||||
.andExpect(jsonPath("$.data.objectKey").value("blobs/session-1"))
|
||||
.andExpect(jsonPath("$.data.status").value("CREATED"))
|
||||
.andExpect(jsonPath("$.data.multipartUpload").value(true))
|
||||
.andExpect(jsonPath("$.data.chunkSize").value(8388608))
|
||||
.andExpect(jsonPath("$.data.chunkCount").value(3));
|
||||
}
|
||||
@@ -85,7 +87,8 @@ class UploadSessionV2ControllerTest {
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value(0))
|
||||
.andExpect(jsonPath("$.data.sessionId").value("session-1"))
|
||||
.andExpect(jsonPath("$.data.status").value("CREATED"));
|
||||
.andExpect(jsonPath("$.data.status").value("CREATED"))
|
||||
.andExpect(jsonPath("$.data.multipartUpload").value(true));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -127,6 +130,29 @@ class UploadSessionV2ControllerTest {
|
||||
.andExpect(jsonPath("$.data.status").value("UPLOADING"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPrepareMultipartPartUploadWithV2Envelope() throws Exception {
|
||||
User user = createUser(7L);
|
||||
when(userDetailsService.loadDomainUser("alice")).thenReturn(user);
|
||||
when(uploadSessionService.prepareOwnedPartUpload(user, "session-1", 1))
|
||||
.thenReturn(new com.yoyuzh.files.storage.PreparedUpload(
|
||||
true,
|
||||
"https://upload.example.com/session-1/part-2",
|
||||
"PUT",
|
||||
Map.of("Content-Type", "video/mp4"),
|
||||
"blobs/session-1"
|
||||
));
|
||||
|
||||
mockMvc.perform(get("/api/v2/files/upload-sessions/session-1/parts/1/prepare")
|
||||
.with(user(userDetails())))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value(0))
|
||||
.andExpect(jsonPath("$.data.direct").value(true))
|
||||
.andExpect(jsonPath("$.data.uploadUrl").value("https://upload.example.com/session-1/part-2"))
|
||||
.andExpect(jsonPath("$.data.method").value("PUT"))
|
||||
.andExpect(jsonPath("$.data.headers['Content-Type']").value("video/mp4"));
|
||||
}
|
||||
|
||||
private UserDetails userDetails() {
|
||||
return org.springframework.security.core.userdetails.User
|
||||
.withUsername("alice")
|
||||
@@ -172,6 +198,7 @@ class UploadSessionV2ControllerTest {
|
||||
session.setContentType("video/mp4");
|
||||
session.setSize(20L * 1024 * 1024);
|
||||
session.setObjectKey("blobs/session-1");
|
||||
session.setMultipartUploadId("upload-123");
|
||||
session.setChunkSize(8L * 1024 * 1024);
|
||||
session.setChunkCount(3);
|
||||
session.setStatus(UploadSessionStatus.CREATED);
|
||||
|
||||
@@ -4,12 +4,12 @@ import com.jayway.jsonpath.JsonPath;
|
||||
import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.FileBlob;
|
||||
import com.yoyuzh.files.FileBlobRepository;
|
||||
import com.yoyuzh.files.FileShareLink;
|
||||
import com.yoyuzh.files.FileShareLinkRepository;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.share.FileShareLink;
|
||||
import com.yoyuzh.files.share.FileShareLinkRepository;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -30,6 +30,8 @@ import static org.springframework.security.test.web.servlet.request.SecurityMock
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
@@ -274,6 +276,120 @@ class ShareV2ControllerIntegrationTest {
|
||||
.andExpect(jsonPath("$.code").value(2404));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDownloadSharedFileAndCountQuota() throws Exception {
|
||||
String createResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"maxDownloads": 1,
|
||||
"allowDownload": true
|
||||
}
|
||||
""".formatted(sharedFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
String token = JsonPath.read(createResponse, "$.data.token");
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", token)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(header().string("Content-Disposition", "attachment; filename*=UTF-8''notes.txt"))
|
||||
.andExpect(content().contentType("text/plain"))
|
||||
.andExpect(content().bytes("hello".getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", token)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isForbidden())
|
||||
.andExpect(jsonPath("$.code").value(2403));
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", token).with(anonymous()))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.downloadCount").value(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectDownloadWhenSharePolicyBlocksIt() throws Exception {
|
||||
String passwordResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"password": "Share123",
|
||||
"allowDownload": true
|
||||
}
|
||||
""".formatted(sharedFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
String passwordToken = JsonPath.read(passwordResponse, "$.data.token");
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", passwordToken)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isBadRequest())
|
||||
.andExpect(jsonPath("$.code").value(2400));
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", passwordToken)
|
||||
.with(anonymous())
|
||||
.param("download", "1")
|
||||
.param("password", "Share123"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(content().bytes("hello".getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
String disabledResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"allowDownload": false
|
||||
}
|
||||
""".formatted(sharedFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
String disabledToken = JsonPath.read(disabledResponse, "$.data.token");
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", disabledToken)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isForbidden())
|
||||
.andExpect(jsonPath("$.code").value(2403));
|
||||
|
||||
String expiredResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"allowDownload": true
|
||||
}
|
||||
""".formatted(sharedFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
String expiredToken = JsonPath.read(expiredResponse, "$.data.token");
|
||||
FileShareLink expiredShare = fileShareLinkRepository.findByToken(expiredToken).orElseThrow();
|
||||
expiredShare.setExpiresAt(LocalDateTime.now().minusMinutes(1));
|
||||
fileShareLinkRepository.save(expiredShare);
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", expiredToken)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isNotFound())
|
||||
.andExpect(jsonPath("$.code").value(2404));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDenyDeletingOtherUsersShare() throws Exception {
|
||||
String createResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
|
||||
@@ -1,16 +1,22 @@
|
||||
package com.yoyuzh.api.v2.tasks;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.BackgroundTask;
|
||||
import com.yoyuzh.files.BackgroundTaskRepository;
|
||||
import com.yoyuzh.files.BackgroundTaskStatus;
|
||||
import com.yoyuzh.files.BackgroundTaskType;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.tasks.BackgroundTask;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskRepository;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskStatus;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskStartupRecovery;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskType;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskWorker;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -19,10 +25,17 @@ import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.not;
|
||||
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.anonymous;
|
||||
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
@@ -55,23 +68,39 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
@Autowired
|
||||
private BackgroundTaskRepository backgroundTaskRepository;
|
||||
|
||||
@Autowired
|
||||
private BackgroundTaskWorker backgroundTaskWorker;
|
||||
|
||||
@Autowired
|
||||
private BackgroundTaskStartupRecovery backgroundTaskStartupRecovery;
|
||||
|
||||
@Autowired
|
||||
private FileBlobRepository fileBlobRepository;
|
||||
|
||||
@Autowired
|
||||
private FileContentStorage fileContentStorage;
|
||||
|
||||
@Autowired
|
||||
private StoredFileRepository storedFileRepository;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
private Long aliceId;
|
||||
private Long archiveDirectoryId;
|
||||
private Long archiveFileId;
|
||||
private Long extractFileId;
|
||||
private Long invalidExtractFileId;
|
||||
private Long unsupportedExtractFileId;
|
||||
private Long mediaFileId;
|
||||
private Long foreignFileId;
|
||||
private Long deletedFileId;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
void setUp() throws Exception {
|
||||
backgroundTaskRepository.deleteAll();
|
||||
storedFileRepository.deleteAll();
|
||||
fileBlobRepository.deleteAll();
|
||||
userRepository.deleteAll();
|
||||
|
||||
User alice = new User();
|
||||
@@ -80,7 +109,8 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
alice.setPhoneNumber("13800138000");
|
||||
alice.setPasswordHash("encoded-password");
|
||||
alice.setCreatedAt(LocalDateTime.now());
|
||||
userRepository.save(alice);
|
||||
alice = userRepository.save(alice);
|
||||
aliceId = alice.getId();
|
||||
|
||||
User bob = new User();
|
||||
bob.setUsername("bob");
|
||||
@@ -91,11 +121,62 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
bob = userRepository.save(bob);
|
||||
|
||||
archiveDirectoryId = storedFileRepository.save(createFile(alice, "/docs", "archive", true, null, 0L, null)).getId();
|
||||
archiveFileId = storedFileRepository.save(createFile(alice, "/docs", "archive-source.txt", false, "text/plain", 12L, null)).getId();
|
||||
extractFileId = storedFileRepository.save(createFile(alice, "/docs", "extract.zip", false, "application/zip", 32L, null)).getId();
|
||||
storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs/archive",
|
||||
"nested.txt",
|
||||
"text/plain",
|
||||
"archive-nested",
|
||||
"nested-content".getBytes(StandardCharsets.UTF_8)
|
||||
));
|
||||
archiveFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs",
|
||||
"archive-source.txt",
|
||||
"text/plain",
|
||||
"archive-source",
|
||||
"archive-source".getBytes(StandardCharsets.UTF_8)
|
||||
)).getId();
|
||||
extractFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs",
|
||||
"extract.zip",
|
||||
"application/zip",
|
||||
"extract-source",
|
||||
createZipArchive(Map.of(
|
||||
"extract/", "",
|
||||
"extract/nested/", "",
|
||||
"extract/notes.txt", "hello",
|
||||
"extract/nested/todo.txt", "world"
|
||||
))
|
||||
)).getId();
|
||||
invalidExtractFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs",
|
||||
"broken.zip",
|
||||
"application/zip",
|
||||
"broken-extract",
|
||||
"not-a-zip".getBytes(StandardCharsets.UTF_8)
|
||||
)).getId();
|
||||
unsupportedExtractFileId = storedFileRepository.save(createFile(alice, "/docs", "backup.7z", false, "application/x-7z-compressed", 64L, null)).getId();
|
||||
mediaFileId = storedFileRepository.save(createFile(alice, "/docs", "media.png", false, "image/png", 24L, null)).getId();
|
||||
foreignFileId = storedFileRepository.save(createFile(bob, "/docs", "foreign.zip", false, "application/zip", 32L, null)).getId();
|
||||
deletedFileId = storedFileRepository.save(createFile(alice, "/docs", "deleted.zip", false, "application/zip", 32L, LocalDateTime.now())).getId();
|
||||
foreignFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
bob,
|
||||
"/docs",
|
||||
"foreign.zip",
|
||||
"application/zip",
|
||||
"foreign-zip",
|
||||
createZipArchive(Map.of("foreign.txt", "blocked"))
|
||||
)).getId();
|
||||
deletedFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs",
|
||||
"deleted.zip",
|
||||
"application/zip",
|
||||
"deleted-zip",
|
||||
createZipArchive(Map.of("deleted.txt", "gone")),
|
||||
LocalDateTime.now()
|
||||
)).getId();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -132,6 +213,9 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"fileId\":" + archiveDirectoryId)))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"path\":\"/docs/archive\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"directory\":true")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":4")))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
@@ -148,6 +232,11 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.type").value("EXTRACT"))
|
||||
.andExpect(jsonPath("$.data.status").value("QUEUED"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputPath\":\"/docs\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputDirectoryName\":\"extract\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
@@ -164,6 +253,9 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.type").value("MEDIA_META"))
|
||||
.andExpect(jsonPath("$.data.status").value("QUEUED"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":2")))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
@@ -187,11 +279,13 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
|
||||
mockMvc.perform(delete("/api/v2/tasks/{id}", extractId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("CANCELLED"));
|
||||
.andExpect(jsonPath("$.data.status").value("CANCELLED"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"cancelled\"")));
|
||||
|
||||
BackgroundTask cancelled = backgroundTaskRepository.findById(extractId).orElseThrow();
|
||||
assertThat(cancelled.getStatus()).isEqualTo(BackgroundTaskStatus.CANCELLED);
|
||||
assertThat(cancelled.getFinishedAt()).isNotNull();
|
||||
assertThat(cancelled.getPublicStateJson()).contains("\"phase\":\"cancelled\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -216,6 +310,289 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
|
||||
mockMvc.perform(delete("/api/v2/tasks/{id}", taskId).with(user("bob")))
|
||||
.andExpect(status().isNotFound());
|
||||
|
||||
mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("bob")))
|
||||
.andExpect(status().isNotFound());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectExtractTaskForNonZipCompatibleArchive() throws Exception {
|
||||
mockMvc.perform(post("/api/v2/tasks/extract")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/backup.7z"
|
||||
}
|
||||
""".formatted(unsupportedExtractFileId)))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCompleteArchiveTaskThroughWorkerAndExposeTerminalState() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/archive")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/archive-source.txt"
|
||||
}
|
||||
""".formatted(archiveFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
String taskResponse = mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("COMPLETED"))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
|
||||
Map<String, Object> publicState = readPublicState(taskResponse);
|
||||
assertThat(publicState).containsEntry("worker", "archive");
|
||||
assertThat(publicState).containsEntry("archivedFilename", "archive-source.txt.zip");
|
||||
assertThat(publicState).containsEntry("archivedPath", "/docs");
|
||||
assertThat(publicState).containsEntry("phase", "completed");
|
||||
assertThat(publicState).containsEntry("attemptCount", 1);
|
||||
assertThat(publicState).containsEntry("maxAttempts", 4);
|
||||
assertThat(publicState).containsEntry("processedFileCount", 1);
|
||||
assertThat(publicState).containsEntry("totalFileCount", 1);
|
||||
assertThat(publicState).containsEntry("processedDirectoryCount", 0);
|
||||
assertThat(publicState).containsEntry("totalDirectoryCount", 0);
|
||||
assertThat(publicState).containsEntry("progressPercent", 100);
|
||||
assertThat(publicState.get("heartbeatAt")).isNotNull();
|
||||
assertThat(publicState).doesNotContainKey("workerOwner");
|
||||
assertThat(publicState).doesNotContainKey("leaseExpiresAt");
|
||||
assertThat(publicState.get("archivedFileId")).isNotNull();
|
||||
assertThat(publicState.get("archiveSize")).isNotNull();
|
||||
|
||||
BackgroundTask completed = backgroundTaskRepository.findById(taskId).orElseThrow();
|
||||
assertThat(completed.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
|
||||
assertThat(completed.getFinishedAt()).isNotNull();
|
||||
assertThat(completed.getErrorMessage()).isNull();
|
||||
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs", "archive-source.txt.zip")).isPresent();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCompleteExtractTaskThroughWorkerAndExposeTerminalState() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/extract")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/extract.zip"
|
||||
}
|
||||
""".formatted(extractFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
String taskResponse = mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("COMPLETED"))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
|
||||
Map<String, Object> publicState = readPublicState(taskResponse);
|
||||
assertThat(publicState).containsEntry("worker", "extract");
|
||||
assertThat(publicState).containsEntry("extractedPath", "/docs/extract");
|
||||
assertThat(publicState).containsEntry("extractedFileCount", 2);
|
||||
assertThat(publicState).containsEntry("extractedDirectoryCount", 2);
|
||||
assertThat(publicState).containsEntry("phase", "completed");
|
||||
assertThat(publicState).containsEntry("attemptCount", 1);
|
||||
assertThat(publicState).containsEntry("maxAttempts", 3);
|
||||
assertThat(publicState).containsEntry("processedFileCount", 2);
|
||||
assertThat(publicState).containsEntry("totalFileCount", 2);
|
||||
assertThat(publicState).containsEntry("processedDirectoryCount", 2);
|
||||
assertThat(publicState).containsEntry("totalDirectoryCount", 2);
|
||||
assertThat(publicState).containsEntry("progressPercent", 100);
|
||||
assertThat(publicState.get("heartbeatAt")).isNotNull();
|
||||
assertThat(publicState).doesNotContainKey("workerOwner");
|
||||
assertThat(publicState).doesNotContainKey("leaseExpiresAt");
|
||||
|
||||
BackgroundTask completed = backgroundTaskRepository.findById(taskId).orElseThrow();
|
||||
assertThat(completed.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
|
||||
assertThat(completed.getFinishedAt()).isNotNull();
|
||||
assertThat(completed.getErrorMessage()).isNull();
|
||||
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs/extract", "notes.txt")).isPresent();
|
||||
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs/extract/nested", "todo.txt")).isPresent();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldMarkExtractTaskFailedWhenWorkerHitsInvalidArchiveContent() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/extract")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/broken.zip"
|
||||
}
|
||||
""".formatted(invalidExtractFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("FAILED"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"failed\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":1")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"failureCategory\":\"DATA_STATE\"")))
|
||||
.andExpect(jsonPath("$.data.errorMessage").value("extract task only supports zip-compatible archives"));
|
||||
|
||||
BackgroundTask failed = backgroundTaskRepository.findById(taskId).orElseThrow();
|
||||
assertThat(failed.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED);
|
||||
assertThat(failed.getFinishedAt()).isNotNull();
|
||||
assertThat(failed.getErrorMessage()).isEqualTo("extract task only supports zip-compatible archives");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRetryFailedTaskAndResetStateToQueued() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/extract")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/broken.zip"
|
||||
}
|
||||
""".formatted(invalidExtractFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
String retryResponse = mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("QUEUED"))
|
||||
.andExpect(jsonPath("$.data.errorMessage").doesNotExist())
|
||||
.andExpect(jsonPath("$.data.finishedAt").doesNotExist())
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
|
||||
Map<String, Object> publicState = readPublicState(retryResponse);
|
||||
assertThat(publicState).containsEntry("phase", "queued");
|
||||
assertThat(publicState).containsEntry("outputPath", "/docs");
|
||||
assertThat(publicState).containsEntry("outputDirectoryName", "broken");
|
||||
assertThat(publicState).containsEntry("attemptCount", 0);
|
||||
assertThat(publicState).containsEntry("maxAttempts", 3);
|
||||
assertThat(publicState).doesNotContainKey("worker");
|
||||
assertThat(publicState).doesNotContainKey("processedFileCount");
|
||||
assertThat(publicState).doesNotContainKey("totalFileCount");
|
||||
|
||||
BackgroundTask retried = backgroundTaskRepository.findById(taskId).orElseThrow();
|
||||
assertThat(retried.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED);
|
||||
assertThat(retried.getFinishedAt()).isNull();
|
||||
assertThat(retried.getErrorMessage()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectRetryForNonFailedTask() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/archive")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/archive-source.txt"
|
||||
}
|
||||
""".formatted(archiveFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
|
||||
mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("alice")))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRecoverOnlyExpiredRunningTaskBackToQueuedOnStartup() throws Exception {
|
||||
BackgroundTask expired = new BackgroundTask();
|
||||
expired.setType(BackgroundTaskType.EXTRACT);
|
||||
expired.setStatus(BackgroundTaskStatus.RUNNING);
|
||||
expired.setUserId(aliceId);
|
||||
expired.setCorrelationId("recover-1");
|
||||
expired.setAttemptCount(1);
|
||||
expired.setMaxAttempts(3);
|
||||
expired.setLeaseOwner("worker-stale");
|
||||
expired.setLeaseExpiresAt(LocalDateTime.now().minusMinutes(2));
|
||||
expired.setHeartbeatAt(LocalDateTime.now().minusMinutes(3));
|
||||
expired.setPublicStateJson("""
|
||||
{"fileId":%d,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-stale","attemptCount":1,"maxAttempts":3}
|
||||
""".formatted(extractFileId));
|
||||
expired.setPrivateStateJson("""
|
||||
{"fileId":%d,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"}
|
||||
""".formatted(extractFileId));
|
||||
expired.setErrorMessage("stale worker");
|
||||
expired.setFinishedAt(LocalDateTime.now());
|
||||
expired = backgroundTaskRepository.save(expired);
|
||||
|
||||
BackgroundTask fresh = new BackgroundTask();
|
||||
fresh.setType(BackgroundTaskType.EXTRACT);
|
||||
fresh.setStatus(BackgroundTaskStatus.RUNNING);
|
||||
fresh.setUserId(aliceId);
|
||||
fresh.setCorrelationId("recover-2");
|
||||
fresh.setAttemptCount(1);
|
||||
fresh.setMaxAttempts(3);
|
||||
fresh.setLeaseOwner("worker-live");
|
||||
fresh.setLeaseExpiresAt(LocalDateTime.now().plusMinutes(5));
|
||||
fresh.setHeartbeatAt(LocalDateTime.now());
|
||||
fresh.setPublicStateJson("""
|
||||
{"fileId":%d,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-live","attemptCount":1,"maxAttempts":3}
|
||||
""".formatted(extractFileId));
|
||||
fresh.setPrivateStateJson("""
|
||||
{"fileId":%d,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"}
|
||||
""".formatted(extractFileId));
|
||||
fresh = backgroundTaskRepository.save(fresh);
|
||||
|
||||
backgroundTaskStartupRecovery.recoverOnStartup();
|
||||
|
||||
mockMvc.perform(get("/api/v2/tasks/{id}", expired.getId()).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("QUEUED"))
|
||||
.andExpect(jsonPath("$.data.errorMessage").doesNotExist())
|
||||
.andExpect(jsonPath("$.data.finishedAt").doesNotExist())
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":1")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputPath\":\"/docs\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputDirectoryName\":\"extract\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"worker\""))))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"workerOwner\""))))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"leaseExpiresAt\""))));
|
||||
|
||||
mockMvc.perform(get("/api/v2/tasks/{id}", fresh.getId()).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("RUNNING"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"workerOwner\":\"worker-live\"")));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -274,4 +651,54 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
file.setDeletedAt(deletedAt);
|
||||
return file;
|
||||
}
|
||||
|
||||
private StoredFile createBlobBackedFile(User user,
|
||||
String path,
|
||||
String filename,
|
||||
String contentType,
|
||||
String objectKeySuffix,
|
||||
byte[] content) {
|
||||
return createBlobBackedFile(user, path, filename, contentType, objectKeySuffix, content, null);
|
||||
}
|
||||
|
||||
private StoredFile createBlobBackedFile(User user,
|
||||
String path,
|
||||
String filename,
|
||||
String contentType,
|
||||
String objectKeySuffix,
|
||||
byte[] content,
|
||||
LocalDateTime deletedAt) {
|
||||
String objectKey = "blobs/test-background-task/" + objectKeySuffix;
|
||||
fileContentStorage.storeBlob(objectKey, contentType, content);
|
||||
|
||||
FileBlob blob = new FileBlob();
|
||||
blob.setObjectKey(objectKey);
|
||||
blob.setContentType(contentType);
|
||||
blob.setSize((long) content.length);
|
||||
blob = fileBlobRepository.save(blob);
|
||||
|
||||
StoredFile file = createFile(user, path, filename, false, contentType, (long) content.length, deletedAt);
|
||||
file.setBlob(blob);
|
||||
return file;
|
||||
}
|
||||
|
||||
private byte[] createZipArchive(Map<String, String> entries) throws IOException {
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) {
|
||||
for (Map.Entry<String, String> entry : entries.entrySet()) {
|
||||
zipOutputStream.putNextEntry(new ZipEntry(entry.getKey()));
|
||||
if (!entry.getKey().endsWith("/")) {
|
||||
zipOutputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
zipOutputStream.closeEntry();
|
||||
}
|
||||
}
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
|
||||
private Map<String, Object> readPublicState(String taskResponse) throws Exception {
|
||||
String publicStateJson = JsonPath.read(taskResponse, "$.data.publicStateJson");
|
||||
return objectMapper.readValue(publicStateJson, new TypeReference<Map<String, Object>>() {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@ import com.yoyuzh.auth.dto.UpdateUserAvatarRequest;
|
||||
import com.yoyuzh.auth.dto.UpdateUserPasswordRequest;
|
||||
import com.yoyuzh.auth.dto.UpdateUserProfileRequest;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.InitiateUploadResponse;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.upload.InitiateUploadResponse;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import com.yoyuzh.files.storage.PreparedUpload;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -39,7 +39,7 @@ class AuthSingleDeviceIntegrationTest {
|
||||
private UserRepository userRepository;
|
||||
|
||||
@Autowired
|
||||
private com.yoyuzh.files.StoredFileRepository storedFileRepository;
|
||||
private com.yoyuzh.files.core.StoredFileRepository storedFileRepository;
|
||||
|
||||
@Autowired
|
||||
private RefreshTokenRepository refreshTokenRepository;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.yoyuzh.auth;
|
||||
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
|
||||
@@ -1,246 +0,0 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.api.v2.ApiV2Exception;
|
||||
import com.yoyuzh.auth.User;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BackgroundTaskServiceTest {
|
||||
|
||||
@Mock
|
||||
private BackgroundTaskRepository backgroundTaskRepository;
|
||||
|
||||
@Mock
|
||||
private StoredFileRepository storedFileRepository;
|
||||
|
||||
private BackgroundTaskService backgroundTaskService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
backgroundTaskService = new BackgroundTaskService(backgroundTaskRepository, storedFileRepository, new ObjectMapper());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationForForeignFile() {
|
||||
User user = createUser(7L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(99L, 7L)).thenReturn(Optional.empty());
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
99L,
|
||||
"/docs/foreign.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("file not found");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationForDeletedFile() {
|
||||
User user = createUser(7L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(100L, 7L)).thenReturn(Optional.empty());
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
100L,
|
||||
"/docs/deleted.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("file not found");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationWhenRequestedPathDoesNotMatchFile() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(11L, user, "/docs", "real.txt", false, "text/plain", 3L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(file));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
11L,
|
||||
"/docs/fake.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("task path does not match file path");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectExtractTaskForDirectory() {
|
||||
User user = createUser(7L);
|
||||
StoredFile directory = createStoredFile(12L, user, "/", "bundle", true, null, 0L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(12L, 7L)).thenReturn(Optional.of(directory));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.EXTRACT,
|
||||
12L,
|
||||
"/bundle",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("task target type is not supported");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectMediaMetadataTaskForNonMediaFile() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(13L, user, "/docs", "notes.txt", false, "text/plain", 9L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(13L, 7L)).thenReturn(Optional.of(file));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.MEDIA_META,
|
||||
13L,
|
||||
"/docs/notes.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("media metadata task only supports media files");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCreateTaskStateFromServerFilePath() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(14L, user, "/docs", "photo.png", false, "image/png", 15L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(14L, 7L)).thenReturn(Optional.of(file));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask task = backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.MEDIA_META,
|
||||
14L,
|
||||
"/docs/photo.png",
|
||||
"media-1"
|
||||
);
|
||||
|
||||
assertThat(task.getPublicStateJson()).contains("\"fileId\":14");
|
||||
assertThat(task.getPublicStateJson()).contains("\"path\":\"/docs/photo.png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"filename\":\"photo.png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"directory\":false");
|
||||
assertThat(task.getPublicStateJson()).contains("\"contentType\":\"image/png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"size\":15");
|
||||
assertThat(task.getPrivateStateJson()).contains("\"taskType\":\"MEDIA_META\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldClaimQueuedTaskOnlyWhenRepositoryTransitionSucceeds() {
|
||||
BackgroundTask task = createTask(1L, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskRepository.claimQueuedTask(
|
||||
eq(1L),
|
||||
eq(BackgroundTaskStatus.QUEUED),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(1L)).thenReturn(Optional.of(task));
|
||||
|
||||
Optional<BackgroundTask> result = backgroundTaskService.claimQueuedTask(1L);
|
||||
|
||||
assertThat(result).containsSame(task);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldNotClaimTaskWhenRepositoryTransitionWasSkipped() {
|
||||
when(backgroundTaskRepository.claimQueuedTask(
|
||||
eq(2L),
|
||||
eq(BackgroundTaskStatus.QUEUED),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
any()
|
||||
)).thenReturn(0);
|
||||
|
||||
Optional<BackgroundTask> result = backgroundTaskService.claimQueuedTask(2L);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCompleteRunningWorkerTaskAndMergePublicState() {
|
||||
BackgroundTask task = createTask(3L, BackgroundTaskStatus.RUNNING);
|
||||
task.setPublicStateJson("{\"fileId\":11}");
|
||||
when(backgroundTaskRepository.findById(3L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskCompleted(3L, Map.of("worker", "noop"));
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
|
||||
assertThat(result.getFinishedAt()).isNotNull();
|
||||
assertThat(result.getErrorMessage()).isNull();
|
||||
assertThat(result.getPublicStateJson()).contains("\"fileId\":11");
|
||||
assertThat(result.getPublicStateJson()).contains("\"worker\":\"noop\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRecordWorkerFailureMessage() {
|
||||
BackgroundTask task = createTask(4L, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskRepository.findById(4L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskFailed(4L, "media parser unavailable");
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED);
|
||||
assertThat(result.getFinishedAt()).isNotNull();
|
||||
assertThat(result.getErrorMessage()).isEqualTo("media parser unavailable");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFindQueuedTaskIdsInCreatedOrderLimit() {
|
||||
BackgroundTask first = createTask(5L, BackgroundTaskStatus.QUEUED);
|
||||
BackgroundTask second = createTask(6L, BackgroundTaskStatus.QUEUED);
|
||||
when(backgroundTaskRepository.findByStatusOrderByCreatedAtAsc(eq(BackgroundTaskStatus.QUEUED), any()))
|
||||
.thenReturn(List.of(first, second));
|
||||
|
||||
List<Long> result = backgroundTaskService.findQueuedTaskIds(2);
|
||||
|
||||
assertThat(result).containsExactly(5L, 6L);
|
||||
}
|
||||
|
||||
private BackgroundTask createTask(Long id, BackgroundTaskStatus status) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setId(id);
|
||||
task.setType(BackgroundTaskType.MEDIA_META);
|
||||
task.setStatus(status);
|
||||
task.setUserId(7L);
|
||||
task.setPublicStateJson("{}");
|
||||
task.setPrivateStateJson("{}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private User createUser(Long id) {
|
||||
User user = new User();
|
||||
user.setId(id);
|
||||
user.setUsername("alice");
|
||||
return user;
|
||||
}
|
||||
|
||||
private StoredFile createStoredFile(Long id,
|
||||
User user,
|
||||
String path,
|
||||
String filename,
|
||||
boolean directory,
|
||||
String contentType,
|
||||
Long size) {
|
||||
StoredFile file = new StoredFile();
|
||||
file.setId(id);
|
||||
file.setUser(user);
|
||||
file.setPath(path);
|
||||
file.setFilename(filename);
|
||||
file.setDirectory(directory);
|
||||
file.setContentType(contentType);
|
||||
file.setSize(size);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BackgroundTaskWorkerTest {
|
||||
|
||||
@Mock
|
||||
private BackgroundTaskService backgroundTaskService;
|
||||
@Mock
|
||||
private BackgroundTaskHandler backgroundTaskHandler;
|
||||
|
||||
private BackgroundTaskWorker backgroundTaskWorker;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
backgroundTaskWorker = new BackgroundTaskWorker(backgroundTaskService, List.of(backgroundTaskHandler));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldClaimAndCompleteQueuedTaskThroughNoopHandler() {
|
||||
BackgroundTask task = createTask(1L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L));
|
||||
when(backgroundTaskService.claimQueuedTask(1L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskHandler.supports(BackgroundTaskType.ARCHIVE)).thenReturn(true);
|
||||
when(backgroundTaskHandler.handle(task)).thenReturn(new BackgroundTaskHandlerResult(Map.of("worker", "noop")));
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
verify(backgroundTaskHandler).handle(task);
|
||||
verify(backgroundTaskService).markWorkerTaskCompleted(1L, Map.of("worker", "noop"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldSkipTaskThatWasNotClaimed() {
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L));
|
||||
when(backgroundTaskService.claimQueuedTask(1L)).thenReturn(Optional.empty());
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isZero();
|
||||
verify(backgroundTaskHandler, never()).handle(org.mockito.ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldMarkTaskFailedWhenHandlerThrows() {
|
||||
BackgroundTask task = createTask(2L, BackgroundTaskType.MEDIA_META, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(2L));
|
||||
when(backgroundTaskService.claimQueuedTask(2L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskHandler.supports(BackgroundTaskType.MEDIA_META)).thenReturn(true);
|
||||
when(backgroundTaskHandler.handle(task)).thenThrow(new IllegalStateException("media parser unavailable"));
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
verify(backgroundTaskService).markWorkerTaskFailed(2L, "media parser unavailable");
|
||||
}
|
||||
|
||||
private BackgroundTask createTask(Long id, BackgroundTaskType type, BackgroundTaskStatus status) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setId(id);
|
||||
task.setType(type);
|
||||
task.setStatus(status);
|
||||
task.setUserId(7L);
|
||||
task.setPublicStateJson("{}");
|
||||
task.setPrivateStateJson("{}");
|
||||
return task;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
@@ -1,6 +1,10 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.policy.StoragePolicy;
|
||||
import com.yoyuzh.files.policy.StoragePolicyCredentialMode;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.files.policy.StoragePolicyType;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user