refactor(files): reorganize backend package layout

This commit is contained in:
yoyuzh
2026-04-09 16:00:34 +08:00
parent da576e0253
commit 3906a523fd
118 changed files with 4722 additions and 978 deletions

View File

@@ -9,13 +9,13 @@ import com.yoyuzh.auth.RefreshTokenService;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.FileBlobRepository; import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.FileService; import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.StoredFile; import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.StoragePolicy; import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.StoragePolicyRepository; import com.yoyuzh.files.policy.StoragePolicyRepository;
import com.yoyuzh.files.StoragePolicyService; import com.yoyuzh.files.policy.StoragePolicyService;
import com.yoyuzh.transfer.OfflineTransferSessionRepository; import com.yoyuzh.transfer.OfflineTransferSessionRepository;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;

View File

@@ -1,8 +1,8 @@
package com.yoyuzh.admin; package com.yoyuzh.admin;
import com.yoyuzh.files.StoragePolicyCapabilities; import com.yoyuzh.files.policy.StoragePolicyCapabilities;
import com.yoyuzh.files.StoragePolicyCredentialMode; import com.yoyuzh.files.policy.StoragePolicyCredentialMode;
import com.yoyuzh.files.StoragePolicyType; import com.yoyuzh.files.policy.StoragePolicyType;
import java.time.LocalDateTime; import java.time.LocalDateTime;

View File

@@ -2,7 +2,7 @@ package com.yoyuzh.api.v2.files;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.files.FileEventService; import com.yoyuzh.files.events.FileEventService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.core.userdetails.UserDetails;

View File

@@ -6,9 +6,9 @@ import com.yoyuzh.api.v2.ApiV2Response;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.FileMetadataResponse; import com.yoyuzh.files.core.FileMetadataResponse;
import com.yoyuzh.files.FileSearchQuery; import com.yoyuzh.files.search.FileSearchQuery;
import com.yoyuzh.files.FileSearchService; import com.yoyuzh.files.search.FileSearchService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.format.annotation.DateTimeFormat; import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.core.annotation.AuthenticationPrincipal;

View File

@@ -0,0 +1,12 @@
package com.yoyuzh.api.v2.files;
import java.util.Map;
public record PreparedUploadV2Response(
boolean direct,
String uploadUrl,
String method,
Map<String, String> headers,
String storageName
) {
}

View File

@@ -3,10 +3,11 @@ package com.yoyuzh.api.v2.files;
import com.yoyuzh.api.v2.ApiV2Response; import com.yoyuzh.api.v2.ApiV2Response;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.files.UploadSession; import com.yoyuzh.files.upload.UploadSession;
import com.yoyuzh.files.UploadSessionCreateCommand; import com.yoyuzh.files.upload.UploadSessionCreateCommand;
import com.yoyuzh.files.UploadSessionPartCommand; import com.yoyuzh.files.upload.UploadSessionPartCommand;
import com.yoyuzh.files.UploadSessionService; import com.yoyuzh.files.upload.UploadSessionService;
import com.yoyuzh.files.storage.PreparedUpload;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.core.annotation.AuthenticationPrincipal;
@@ -77,10 +78,26 @@ public class UploadSessionV2Controller {
return ApiV2Response.success(toResponse(session)); return ApiV2Response.success(toResponse(session));
} }
@GetMapping("/{sessionId}/parts/{partIndex}/prepare")
public ApiV2Response<PreparedUploadV2Response> preparePartUpload(@AuthenticationPrincipal UserDetails userDetails,
@PathVariable String sessionId,
@PathVariable int partIndex) {
User user = userDetailsService.loadDomainUser(userDetails.getUsername());
PreparedUpload preparedUpload = uploadSessionService.prepareOwnedPartUpload(user, sessionId, partIndex);
return ApiV2Response.success(new PreparedUploadV2Response(
preparedUpload.direct(),
preparedUpload.uploadUrl(),
preparedUpload.method(),
preparedUpload.headers(),
preparedUpload.storageName()
));
}
private UploadSessionV2Response toResponse(UploadSession session) { private UploadSessionV2Response toResponse(UploadSession session) {
return new UploadSessionV2Response( return new UploadSessionV2Response(
session.getSessionId(), session.getSessionId(),
session.getObjectKey(), session.getObjectKey(),
session.getMultipartUploadId() != null,
session.getTargetPath(), session.getTargetPath(),
session.getFilename(), session.getFilename(),
session.getContentType(), session.getContentType(),

View File

@@ -5,6 +5,7 @@ import java.time.LocalDateTime;
public record UploadSessionV2Response( public record UploadSessionV2Response(
String sessionId, String sessionId,
String objectKey, String objectKey,
boolean multipartUpload,
String path, String path,
String filename, String filename,
String contentType, String contentType,

View File

@@ -4,14 +4,15 @@ import com.yoyuzh.api.v2.ApiV2Response;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.FileMetadataResponse; import com.yoyuzh.files.core.FileMetadataResponse;
import com.yoyuzh.files.ShareV2Service; import com.yoyuzh.files.share.ShareV2Service;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Max;
import jakarta.validation.constraints.Min; import jakarta.validation.constraints.Min;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
@@ -43,6 +44,12 @@ public class ShareV2Controller {
return ApiV2Response.success(shareV2Service.getShare(token)); return ApiV2Response.success(shareV2Service.getShare(token));
} }
@GetMapping(value = "/{token}", params = "download")
public ResponseEntity<?> downloadShare(@PathVariable String token,
@RequestParam(required = false) String password) {
return shareV2Service.downloadSharedFile(token, password);
}
@PostMapping("/{token}/verify-password") @PostMapping("/{token}/verify-password")
public ApiV2Response<ShareV2Response> verifyPassword(@PathVariable String token, public ApiV2Response<ShareV2Response> verifyPassword(@PathVariable String token,
@Valid @RequestBody VerifySharePasswordV2Request request) { @Valid @RequestBody VerifySharePasswordV2Request request) {

View File

@@ -1,6 +1,6 @@
package com.yoyuzh.api.v2.shares; package com.yoyuzh.api.v2.shares;
import com.yoyuzh.files.FileMetadataResponse; import com.yoyuzh.files.core.FileMetadataResponse;
import java.time.LocalDateTime; import java.time.LocalDateTime;

View File

@@ -1,7 +1,7 @@
package com.yoyuzh.api.v2.tasks; package com.yoyuzh.api.v2.tasks;
import com.yoyuzh.files.BackgroundTaskStatus; import com.yoyuzh.files.tasks.BackgroundTaskStatus;
import com.yoyuzh.files.BackgroundTaskType; import com.yoyuzh.files.tasks.BackgroundTaskType;
import java.time.LocalDateTime; import java.time.LocalDateTime;

View File

@@ -4,9 +4,9 @@ import com.yoyuzh.api.v2.ApiV2Response;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.BackgroundTask; import com.yoyuzh.files.tasks.BackgroundTask;
import com.yoyuzh.files.BackgroundTaskService; import com.yoyuzh.files.tasks.BackgroundTaskService;
import com.yoyuzh.files.BackgroundTaskType; import com.yoyuzh.files.tasks.BackgroundTaskType;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Max;
import jakarta.validation.constraints.Min; import jakarta.validation.constraints.Min;
@@ -63,6 +63,13 @@ public class BackgroundTaskV2Controller {
return ApiV2Response.success(toResponse(backgroundTaskService.cancelOwnedTask(user, id))); return ApiV2Response.success(toResponse(backgroundTaskService.cancelOwnedTask(user, id)));
} }
@PostMapping("/{id}/retry")
public ApiV2Response<BackgroundTaskResponse> retry(@AuthenticationPrincipal UserDetails userDetails,
@PathVariable Long id) {
User user = userDetailsService.loadDomainUser(userDetails.getUsername());
return ApiV2Response.success(toResponse(backgroundTaskService.retryOwnedTask(user, id)));
}
@PostMapping("/archive") @PostMapping("/archive")
public ApiV2Response<BackgroundTaskResponse> createArchiveTask(@AuthenticationPrincipal UserDetails userDetails, public ApiV2Response<BackgroundTaskResponse> createArchiveTask(@AuthenticationPrincipal UserDetails userDetails,
@Valid @RequestBody CreateBackgroundTaskRequest request) { @Valid @RequestBody CreateBackgroundTaskRequest request) {

View File

@@ -9,8 +9,8 @@ import com.yoyuzh.auth.dto.UpdateUserProfileRequest;
import com.yoyuzh.auth.dto.UserProfileResponse; import com.yoyuzh.auth.dto.UserProfileResponse;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.FileService; import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.InitiateUploadResponse; import com.yoyuzh.files.upload.InitiateUploadResponse;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;

View File

@@ -1,7 +1,7 @@
package com.yoyuzh.auth; package com.yoyuzh.auth;
import com.yoyuzh.files.FileService; import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.boot.CommandLineRunner; import org.springframework.boot.CommandLineRunner;
import org.springframework.context.annotation.Profile; import org.springframework.context.annotation.Profile;

View File

@@ -1,8 +0,0 @@
package com.yoyuzh.files;
public interface BackgroundTaskHandler {
boolean supports(BackgroundTaskType type);
BackgroundTaskHandlerResult handle(BackgroundTask task);
}

View File

@@ -1,35 +0,0 @@
package com.yoyuzh.files;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
public interface BackgroundTaskRepository extends JpaRepository<BackgroundTask, Long> {
Page<BackgroundTask> findByUserIdOrderByCreatedAtDesc(Long userId, Pageable pageable);
Optional<BackgroundTask> findByIdAndUserId(Long id, Long userId);
List<BackgroundTask> findByStatusOrderByCreatedAtAsc(BackgroundTaskStatus status, Pageable pageable);
@Modifying
@Query("""
update BackgroundTask task
set task.status = :runningStatus,
task.errorMessage = null,
task.updatedAt = :updatedAt
where task.id = :id
and task.status = :queuedStatus
""")
int claimQueuedTask(@Param("id") Long id,
@Param("queuedStatus") BackgroundTaskStatus queuedStatus,
@Param("runningStatus") BackgroundTaskStatus runningStatus,
@Param("updatedAt") LocalDateTime updatedAt);
}

View File

@@ -1,314 +0,0 @@
package com.yoyuzh.files;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.api.v2.ApiV2ErrorCode;
import com.yoyuzh.api.v2.ApiV2Exception;
import com.yoyuzh.auth.User;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
@Service
@RequiredArgsConstructor
public class BackgroundTaskService {
private static final List<String> ARCHIVE_EXTENSIONS = List.of(
".zip", ".jar", ".war", ".7z", ".rar", ".tar", ".gz", ".tgz", ".bz2", ".xz"
);
private static final List<String> MEDIA_EXTENSIONS = List.of(
".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg",
".mp4", ".mov", ".mkv", ".webm", ".avi",
".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"
);
private final BackgroundTaskRepository backgroundTaskRepository;
private final StoredFileRepository storedFileRepository;
private final ObjectMapper objectMapper;
@Transactional
public BackgroundTask createQueuedFileTask(User user,
BackgroundTaskType type,
Long fileId,
String requestedPath,
String correlationId) {
StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, user.getId())
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "file not found"));
String logicalPath = buildLogicalPath(file);
if (!logicalPath.equals(normalizeLogicalPath(requestedPath))) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task path does not match file path");
}
validateTaskTarget(type, file);
Map<String, Object> publicState = fileState(file, logicalPath);
Map<String, Object> privateState = new LinkedHashMap<>(publicState);
privateState.put("taskType", type.name());
return createQueuedTask(user, type, publicState, privateState, correlationId);
}
@Transactional
public BackgroundTask createQueuedTask(User user,
BackgroundTaskType type,
Map<String, Object> publicState,
Map<String, Object> privateState,
String correlationId) {
BackgroundTask task = new BackgroundTask();
task.setUserId(user.getId());
task.setType(type);
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setPublicStateJson(toJson(publicState));
task.setPrivateStateJson(toJson(privateState));
task.setCorrelationId(normalizeCorrelationId(correlationId));
return backgroundTaskRepository.save(task);
}
public Page<BackgroundTask> listOwnedTasks(User user, Pageable pageable) {
return backgroundTaskRepository.findByUserIdOrderByCreatedAtDesc(user.getId(), pageable);
}
public BackgroundTask getOwnedTask(User user, Long id) {
return backgroundTaskRepository.findByIdAndUserId(id, user.getId())
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
}
@Transactional
public BackgroundTask cancelOwnedTask(User user, Long id) {
BackgroundTask task = getOwnedTask(user, id);
if (task.isTerminal()) {
return task;
}
if (task.getStatus() == BackgroundTaskStatus.QUEUED || task.getStatus() == BackgroundTaskStatus.RUNNING) {
task.setStatus(BackgroundTaskStatus.CANCELLED);
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
return task;
}
@Transactional
public BackgroundTask markRunning(User user, Long id) {
BackgroundTask task = getOwnedTask(user, id);
if (task.isTerminal()) {
return task;
}
task.setStatus(BackgroundTaskStatus.RUNNING);
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markCompleted(User user, Long id) {
BackgroundTask task = getOwnedTask(user, id);
if (task.isTerminal()) {
return task;
}
task.setStatus(BackgroundTaskStatus.COMPLETED);
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markFailed(User user, Long id, String errorMessage) {
BackgroundTask task = getOwnedTask(user, id);
if (task.isTerminal()) {
return task;
}
task.setStatus(BackgroundTaskStatus.FAILED);
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed");
return backgroundTaskRepository.save(task);
}
public List<Long> findQueuedTaskIds(int limit) {
if (limit <= 0) {
return List.of();
}
return backgroundTaskRepository.findByStatusOrderByCreatedAtAsc(
BackgroundTaskStatus.QUEUED,
PageRequest.of(0, limit)
)
.stream()
.map(BackgroundTask::getId)
.toList();
}
@Transactional
public Optional<BackgroundTask> claimQueuedTask(Long id) {
int claimed = backgroundTaskRepository.claimQueuedTask(
id,
BackgroundTaskStatus.QUEUED,
BackgroundTaskStatus.RUNNING,
LocalDateTime.now()
);
if (claimed != 1) {
return Optional.empty();
}
return backgroundTaskRepository.findById(id);
}
@Transactional
public BackgroundTask markWorkerTaskCompleted(Long id, Map<String, Object> publicStatePatch) {
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
if (task.isTerminal() || task.getStatus() != BackgroundTaskStatus.RUNNING) {
return task;
}
Map<String, Object> nextPublicState = parseJsonObject(task.getPublicStateJson());
if (publicStatePatch != null) {
nextPublicState.putAll(publicStatePatch);
}
task.setPublicStateJson(toJson(nextPublicState));
task.setStatus(BackgroundTaskStatus.COMPLETED);
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markWorkerTaskFailed(Long id, String errorMessage) {
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
if (task.isTerminal()) {
return task;
}
task.setStatus(BackgroundTaskStatus.FAILED);
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed");
return backgroundTaskRepository.save(task);
}
private String normalizeCorrelationId(String correlationId) {
if (StringUtils.hasText(correlationId)) {
return correlationId.trim();
}
return UUID.randomUUID().toString().replace("-", "");
}
private void validateTaskTarget(BackgroundTaskType type, StoredFile file) {
if (type == BackgroundTaskType.ARCHIVE) {
return;
}
if (file.isDirectory()) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task target type is not supported");
}
if (type == BackgroundTaskType.EXTRACT && !isArchiveLike(file)) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "extract task only supports archive files");
}
if (type == BackgroundTaskType.MEDIA_META && !isMediaLike(file)) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "media metadata task only supports media files");
}
}
private Map<String, Object> fileState(StoredFile file, String logicalPath) {
Map<String, Object> state = new LinkedHashMap<>();
state.put("fileId", file.getId());
state.put("path", logicalPath);
state.put("filename", file.getFilename());
state.put("directory", file.isDirectory());
state.put("contentType", file.getContentType());
state.put("size", file.getSize());
return state;
}
private boolean isArchiveLike(StoredFile file) {
String contentType = normalizeContentType(file.getContentType());
if (contentType.contains("zip")
|| contentType.contains("x-tar")
|| contentType.contains("gzip")
|| contentType.contains("x-7z")
|| contentType.contains("x-rar")
|| contentType.contains("java-archive")) {
return true;
}
return hasExtension(file.getFilename(), ARCHIVE_EXTENSIONS);
}
private boolean isMediaLike(StoredFile file) {
String contentType = normalizeContentType(file.getContentType());
if (contentType.startsWith("image/") || contentType.startsWith("video/") || contentType.startsWith("audio/")) {
return true;
}
return hasExtension(file.getFilename(), MEDIA_EXTENSIONS);
}
private boolean hasExtension(String filename, List<String> extensions) {
if (!StringUtils.hasText(filename)) {
return false;
}
String normalized = filename.toLowerCase(Locale.ROOT);
return extensions.stream().anyMatch(normalized::endsWith);
}
private String normalizeContentType(String contentType) {
if (!StringUtils.hasText(contentType)) {
return "";
}
return contentType.trim().toLowerCase(Locale.ROOT);
}
private String buildLogicalPath(StoredFile file) {
String parent = normalizeLogicalPath(file.getPath());
if ("/".equals(parent)) {
return "/" + file.getFilename();
}
return parent + "/" + file.getFilename();
}
private String normalizeLogicalPath(String path) {
if (!StringUtils.hasText(path)) {
return "/";
}
String normalized = path.trim().replace('\\', '/');
while (normalized.contains("//")) {
normalized = normalized.replace("//", "/");
}
if (!normalized.startsWith("/")) {
normalized = "/" + normalized;
}
while (normalized.length() > 1 && normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
private String toJson(Map<String, Object> value) {
Map<String, Object> safeValue = value == null ? new LinkedHashMap<>() : new LinkedHashMap<>(value);
try {
return objectMapper.writeValueAsString(safeValue);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to serialize background task state", ex);
}
}
private Map<String, Object> parseJsonObject(String value) {
if (!StringUtils.hasText(value)) {
return new LinkedHashMap<>();
}
try {
return objectMapper.readValue(value, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to parse background task state", ex);
}
}
}

View File

@@ -1,61 +0,0 @@
package com.yoyuzh.files;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
public class BackgroundTaskWorker {
private static final int DEFAULT_BATCH_SIZE = 5;
private final BackgroundTaskService backgroundTaskService;
private final List<BackgroundTaskHandler> handlers;
public BackgroundTaskWorker(BackgroundTaskService backgroundTaskService,
List<BackgroundTaskHandler> handlers) {
this.backgroundTaskService = backgroundTaskService;
this.handlers = List.copyOf(handlers);
}
@Scheduled(
fixedDelayString = "${app.background-tasks.worker.fixed-delay-ms:30000}",
initialDelayString = "${app.background-tasks.worker.initial-delay-ms:30000}"
)
public void runScheduledBatch() {
processQueuedTasks(DEFAULT_BATCH_SIZE);
}
public int processQueuedTasks(int maxTasks) {
int processedCount = 0;
for (Long taskId : backgroundTaskService.findQueuedTaskIds(maxTasks)) {
var claimedTask = backgroundTaskService.claimQueuedTask(taskId);
if (claimedTask.isEmpty()) {
continue;
}
execute(claimedTask.get());
processedCount += 1;
}
return processedCount;
}
private void execute(BackgroundTask task) {
try {
BackgroundTaskHandler handler = findHandler(task);
BackgroundTaskHandlerResult result = handler.handle(task);
backgroundTaskService.markWorkerTaskCompleted(task.getId(), result.publicStatePatch());
} catch (Exception ex) {
String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage();
backgroundTaskService.markWorkerTaskFailed(task.getId(), message);
}
}
private BackgroundTaskHandler findHandler(BackgroundTask task) {
return handlers.stream()
.filter(handler -> handler.supports(task.getType()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("No background task handler for " + task.getType()));
}
}

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
public record DownloadUrlResponse(String url) { public record DownloadUrlResponse(String url) {
} }

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query; import org.springframework.data.jpa.repository.Query;

View File

@@ -1,8 +1,14 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.common.ApiResponse; import com.yoyuzh.common.ApiResponse;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.share.CreateFileShareLinkResponse;
import com.yoyuzh.files.share.FileShareDetailsResponse;
import com.yoyuzh.files.share.ImportSharedFileRequest;
import com.yoyuzh.files.upload.CompleteUploadRequest;
import com.yoyuzh.files.upload.InitiateUploadRequest;
import com.yoyuzh.files.upload.InitiateUploadResponse;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import jakarta.persistence.Column; import jakarta.persistence.Column;

View File

@@ -1,5 +1,6 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import com.yoyuzh.files.policy.StoragePolicyService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.boot.CommandLineRunner; import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order; import org.springframework.core.annotation.Order;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
public enum FileEntityType { public enum FileEntityType {
VERSION, VERSION,

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import java.time.LocalDateTime; import java.time.LocalDateTime;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import com.yoyuzh.admin.AdminMetricsService; import com.yoyuzh.admin.AdminMetricsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
@@ -6,8 +6,18 @@ import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.config.FileStorageProperties; import com.yoyuzh.config.FileStorageProperties;
import com.yoyuzh.files.events.FileEventService;
import com.yoyuzh.files.events.FileEventType;
import com.yoyuzh.files.policy.StoragePolicyService;
import com.yoyuzh.files.share.CreateFileShareLinkResponse;
import com.yoyuzh.files.share.FileShareDetailsResponse;
import com.yoyuzh.files.share.FileShareLink;
import com.yoyuzh.files.share.FileShareLinkRepository;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import com.yoyuzh.files.storage.PreparedUpload; import com.yoyuzh.files.storage.PreparedUpload;
import com.yoyuzh.files.upload.CompleteUploadRequest;
import com.yoyuzh.files.upload.InitiateUploadRequest;
import com.yoyuzh.files.upload.InitiateUploadResponse;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;
@@ -20,6 +30,7 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
@@ -41,6 +52,7 @@ import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.Locale; import java.util.Locale;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
@Service @Service
@@ -566,36 +578,49 @@ public class FileService {
}); });
} }
private ResponseEntity<byte[]> downloadDirectory(User user, StoredFile directory) { @Transactional
String logicalPath = buildLogicalPath(directory); public void importExternalFilesAtomically(User recipient,
String archiveName = directory.getFilename() + ".zip"; List<String> directories,
List<StoredFile> descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(user.getId(), logicalPath) List<ExternalFileImport> files) {
.stream() importExternalFilesAtomically(recipient, directories, files, null);
.sorted(Comparator.comparing(StoredFile::getPath).thenComparing(StoredFile::getFilename)) }
.toList();
byte[] archiveBytes; @Transactional
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); public void importExternalFilesAtomically(User recipient,
ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) { List<String> directories,
Set<String> createdEntries = new LinkedHashSet<>(); List<ExternalFileImport> files,
writeDirectoryEntry(zipOutputStream, createdEntries, directory.getFilename() + "/"); ExternalImportProgressListener progressListener) {
List<String> normalizedDirectories = normalizeExternalImportDirectories(directories);
List<ExternalFileImport> normalizedFiles = normalizeExternalImportFiles(files);
validateExternalImportBatch(recipient, normalizedDirectories, normalizedFiles);
for (StoredFile descendant : descendants) { List<String> writtenBlobObjectKeys = new ArrayList<>();
String entryName = buildZipEntryName(directory.getFilename(), logicalPath, descendant); int totalDirectoryCount = normalizedDirectories.size();
if (descendant.isDirectory()) { int totalFileCount = normalizedFiles.size();
writeDirectoryEntry(zipOutputStream, createdEntries, entryName + "/"); int processedDirectoryCount = 0;
continue; int processedFileCount = 0;
} try {
for (String directory : normalizedDirectories) {
ensureParentDirectoryEntries(zipOutputStream, createdEntries, entryName); mkdir(recipient, directory);
writeFileEntry(zipOutputStream, createdEntries, entryName, processedDirectoryCount += 1;
fileContentStorage.readBlob(getRequiredBlob(descendant).getObjectKey())); reportExternalImportProgress(progressListener, processedFileCount, totalFileCount,
processedDirectoryCount, totalDirectoryCount);
} }
zipOutputStream.finish(); for (ExternalFileImport file : normalizedFiles) {
archiveBytes = outputStream.toByteArray(); storeExternalImportFile(recipient, file, writtenBlobObjectKeys);
} catch (IOException ex) { processedFileCount += 1;
throw new BusinessException(ErrorCode.UNKNOWN, "目录压缩失败"); reportExternalImportProgress(progressListener, processedFileCount, totalFileCount,
processedDirectoryCount, totalDirectoryCount);
}
} catch (RuntimeException ex) {
cleanupWrittenBlobs(writtenBlobObjectKeys, ex);
throw ex;
} }
}
private ResponseEntity<byte[]> downloadDirectory(User user, StoredFile directory) {
String archiveName = directory.getFilename() + ".zip";
byte[] archiveBytes = buildArchiveBytes(directory);
return ResponseEntity.ok() return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, .header(HttpHeaders.CONTENT_DISPOSITION,
@@ -604,6 +629,63 @@ public class FileService {
.body(archiveBytes); .body(archiveBytes);
} }
public byte[] buildArchiveBytes(StoredFile source) {
return buildArchiveBytes(source, null);
}
public byte[] buildArchiveBytes(StoredFile source, ArchiveBuildProgressListener progressListener) {
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) {
Set<String> createdEntries = new LinkedHashSet<>();
ArchiveBuildProgressState progressState = createArchiveBuildProgressState(source, progressListener);
reportArchiveProgress(progressState);
if (source.isDirectory()) {
writeDirectoryArchiveEntries(zipOutputStream, createdEntries, source, progressState);
} else {
writeFileArchiveEntry(zipOutputStream, createdEntries, source.getFilename(), source, progressState);
}
zipOutputStream.finish();
return outputStream.toByteArray();
} catch (IOException ex) {
throw new BusinessException(ErrorCode.UNKNOWN, "目录压缩失败");
}
}
public ZipCompatibleArchive readZipCompatibleArchive(StoredFile source) {
byte[] archiveBytes = fileContentStorage.readBlob(getRequiredBlob(source).getObjectKey());
try (ZipInputStream zipInputStream = new ZipInputStream(
new ByteArrayInputStream(archiveBytes),
StandardCharsets.UTF_8)) {
List<ZipCompatibleArchiveEntry> entries = new ArrayList<>();
Map<String, Boolean> seenEntries = new HashMap<>();
ZipEntry entry = zipInputStream.getNextEntry();
while (entry != null) {
String relativePath = normalizeZipCompatibleEntryPath(entry.getName());
if (StringUtils.hasText(relativePath)) {
boolean directory = entry.isDirectory() || entry.getName().endsWith("/");
Boolean existingType = seenEntries.putIfAbsent(relativePath, directory);
if (existingType != null) {
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法");
}
entries.add(new ZipCompatibleArchiveEntry(
relativePath,
directory,
directory ? new byte[0] : zipInputStream.readAllBytes()
));
}
entry = zipInputStream.getNextEntry();
}
if (entries.isEmpty() && !hasZipCompatibleSignature(archiveBytes)) {
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包读取失败");
}
return new ZipCompatibleArchive(entries, detectCommonRootDirectoryName(entries));
} catch (BusinessException ex) {
throw ex;
} catch (IOException ex) {
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包读取失败");
}
}
private boolean shouldUsePublicPackageDownload(StoredFile storedFile) { private boolean shouldUsePublicPackageDownload(StoredFile storedFile) {
return fileContentStorage.supportsDirectDownload() return fileContentStorage.supportsDirectDownload()
&& StringUtils.hasText(packageDownloadBaseUrl) && StringUtils.hasText(packageDownloadBaseUrl)
@@ -854,6 +936,62 @@ public class FileService {
ensureWithinStorageQuota(user, size); ensureWithinStorageQuota(user, size);
} }
private List<String> normalizeExternalImportDirectories(List<String> directories) {
if (directories == null || directories.isEmpty()) {
return List.of();
}
return directories.stream()
.map(this::normalizeDirectoryPath)
.distinct()
.sorted(Comparator.comparingInt(String::length).thenComparing(String::compareTo))
.toList();
}
private List<ExternalFileImport> normalizeExternalImportFiles(List<ExternalFileImport> files) {
if (files == null || files.isEmpty()) {
return List.of();
}
return files.stream()
.map(file -> new ExternalFileImport(
normalizeDirectoryPath(file.path()),
normalizeLeafName(file.filename()),
StringUtils.hasText(file.contentType()) ? file.contentType().trim() : "application/octet-stream",
file.content() == null ? new byte[0] : file.content()
))
.toList();
}
private void validateExternalImportBatch(User recipient,
List<String> directories,
List<ExternalFileImport> files) {
ensureWithinStorageQuota(recipient, files.stream().mapToLong(ExternalFileImport::size).sum());
Set<String> plannedTargets = new LinkedHashSet<>();
for (String directory : directories) {
if ("/".equals(directory)) {
continue;
}
if (!plannedTargets.add(directory)) {
continue;
}
String parentPath = extractParentPath(directory);
String directoryName = extractLeafName(directory);
if (storedFileRepository.existsByUserIdAndPathAndFilename(recipient.getId(), parentPath, directoryName)) {
throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在");
}
}
for (ExternalFileImport file : files) {
String logicalPath = buildTargetLogicalPath(file.path(), file.filename());
if (plannedTargets.contains(logicalPath) || !plannedTargets.add(logicalPath)) {
throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在");
}
if (storedFileRepository.existsByUserIdAndPathAndFilename(recipient.getId(), file.path(), file.filename())) {
throw new BusinessException(ErrorCode.UNKNOWN, "同目录下文件已存在");
}
}
}
private void ensureWithinStorageQuota(User user, long additionalBytes) { private void ensureWithinStorageQuota(User user, long additionalBytes) {
if (additionalBytes <= 0) { if (additionalBytes <= 0) {
return; return;
@@ -900,6 +1038,25 @@ public class FileService {
} }
} }
private void storeExternalImportFile(User recipient,
ExternalFileImport file,
List<String> writtenBlobObjectKeys) {
validateUpload(recipient, file.path(), file.filename(), file.size());
ensureDirectoryHierarchy(recipient, file.path());
String objectKey = createBlobObjectKey();
writtenBlobObjectKeys.add(objectKey);
fileContentStorage.storeBlob(objectKey, file.contentType(), file.content());
FileBlob blob = createAndSaveBlob(objectKey, file.contentType(), file.size());
saveFileMetadata(
recipient,
file.path(),
file.filename(),
file.contentType(),
file.size(),
blob
);
}
private void moveToRecycleBin(List<StoredFile> filesToRecycle, Long recycleRootId) { private void moveToRecycleBin(List<StoredFile> filesToRecycle, Long recycleRootId) {
if (filesToRecycle.isEmpty()) { if (filesToRecycle.isEmpty()) {
return; return;
@@ -1071,6 +1228,37 @@ public class FileService {
return savedFile; return savedFile;
} }
private void writeDirectoryArchiveEntries(ZipOutputStream zipOutputStream,
Set<String> createdEntries,
StoredFile directory,
ArchiveBuildProgressState progressState) throws IOException {
String logicalPath = buildLogicalPath(directory);
List<StoredFile> descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(directory.getUser().getId(), logicalPath)
.stream()
.sorted(Comparator.comparing(StoredFile::getPath).thenComparing(StoredFile::getFilename))
.toList();
writeDirectoryEntry(zipOutputStream, createdEntries, directory.getFilename() + "/", progressState);
for (StoredFile descendant : descendants) {
String entryName = buildZipEntryName(directory.getFilename(), logicalPath, descendant);
if (descendant.isDirectory()) {
writeDirectoryEntry(zipOutputStream, createdEntries, entryName + "/", progressState);
continue;
}
writeFileArchiveEntry(zipOutputStream, createdEntries, entryName, descendant, progressState);
}
}
private void writeFileArchiveEntry(ZipOutputStream zipOutputStream,
Set<String> createdEntries,
String entryName,
StoredFile file,
ArchiveBuildProgressState progressState) throws IOException {
ensureParentDirectoryEntries(zipOutputStream, createdEntries, entryName, progressState);
writeFileEntry(zipOutputStream, createdEntries, entryName, progressState,
fileContentStorage.readBlob(getRequiredBlob(file).getObjectKey()));
}
private String buildZipEntryName(String rootDirectoryName, String rootLogicalPath, StoredFile storedFile) { private String buildZipEntryName(String rootDirectoryName, String rootLogicalPath, StoredFile storedFile) {
StringBuilder entryName = new StringBuilder(rootDirectoryName).append('/'); StringBuilder entryName = new StringBuilder(rootDirectoryName).append('/');
if (!storedFile.getPath().equals(rootLogicalPath)) { if (!storedFile.getPath().equals(rootLogicalPath)) {
@@ -1080,24 +1268,153 @@ public class FileService {
return entryName.toString(); return entryName.toString();
} }
private void ensureParentDirectoryEntries(ZipOutputStream zipOutputStream, Set<String> createdEntries, String entryName) throws IOException { private String normalizeZipCompatibleEntryPath(String entryName) {
String normalized = entryName == null ? "" : entryName.trim().replace("\\", "/");
if (!StringUtils.hasText(normalized)) {
return "";
}
if (normalized.startsWith("/")) {
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法");
}
while (normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
if (!StringUtils.hasText(normalized)) {
return "";
}
StringBuilder sanitized = new StringBuilder();
for (String segment : normalized.split("/")) {
if (!StringUtils.hasText(segment) || ".".equals(segment) || "..".equals(segment)) {
throw new BusinessException(ErrorCode.UNKNOWN, "压缩包内容不合法");
}
if (sanitized.length() > 0) {
sanitized.append('/');
}
sanitized.append(normalizeLeafName(segment));
}
return sanitized.toString();
}
private String detectCommonRootDirectoryName(List<ZipCompatibleArchiveEntry> entries) {
String candidate = null;
boolean hasNestedEntry = false;
boolean hasDirectoryCandidate = false;
for (ZipCompatibleArchiveEntry entry : entries) {
String relativePath = entry.relativePath();
int slashIndex = relativePath.indexOf('/');
String topSegment = slashIndex >= 0 ? relativePath.substring(0, slashIndex) : relativePath;
if (candidate == null) {
candidate = topSegment;
} else if (!candidate.equals(topSegment)) {
return null;
}
if (slashIndex >= 0) {
hasNestedEntry = true;
}
if (entry.directory() && candidate.equals(relativePath)) {
hasDirectoryCandidate = true;
}
if (!entry.directory() && candidate.equals(relativePath)) {
return null;
}
}
if (!hasNestedEntry && !hasDirectoryCandidate) {
return null;
}
return candidate;
}
private boolean hasZipCompatibleSignature(byte[] archiveBytes) {
if (archiveBytes == null || archiveBytes.length < 4) {
return false;
}
return archiveBytes[0] == 'P'
&& archiveBytes[1] == 'K'
&& (archiveBytes[2] == 3 || archiveBytes[2] == 5 || archiveBytes[2] == 7)
&& (archiveBytes[3] == 4 || archiveBytes[3] == 6 || archiveBytes[3] == 8);
}
public ArchiveSourceSummary summarizeArchiveSource(StoredFile source) {
if (!source.isDirectory()) {
return new ArchiveSourceSummary(1, 0);
}
String logicalPath = buildLogicalPath(source);
List<StoredFile> descendants = storedFileRepository.findByUserIdAndPathEqualsOrDescendant(source.getUser().getId(), logicalPath);
int directoryCount = 1 + (int) descendants.stream().filter(StoredFile::isDirectory).count();
int fileCount = (int) descendants.stream().filter(file -> !file.isDirectory()).count();
return new ArchiveSourceSummary(fileCount, directoryCount);
}
private ArchiveBuildProgressState createArchiveBuildProgressState(StoredFile source,
ArchiveBuildProgressListener progressListener) {
if (progressListener == null) {
return null;
}
ArchiveSourceSummary summary = summarizeArchiveSource(source);
return new ArchiveBuildProgressState(progressListener, summary.fileCount(), summary.directoryCount());
}
private void reportArchiveProgress(ArchiveBuildProgressState progressState) {
if (progressState == null) {
return;
}
progressState.listener.onProgress(new ArchiveBuildProgress(
progressState.processedFileCount,
progressState.totalFileCount,
progressState.processedDirectoryCount,
progressState.totalDirectoryCount
));
}
private void reportExternalImportProgress(ExternalImportProgressListener progressListener,
int processedFileCount,
int totalFileCount,
int processedDirectoryCount,
int totalDirectoryCount) {
if (progressListener == null) {
return;
}
progressListener.onProgress(new ExternalImportProgress(
processedFileCount,
totalFileCount,
processedDirectoryCount,
totalDirectoryCount
));
}
private void ensureParentDirectoryEntries(ZipOutputStream zipOutputStream,
Set<String> createdEntries,
String entryName,
ArchiveBuildProgressState progressState) throws IOException {
int slashIndex = entryName.indexOf('/'); int slashIndex = entryName.indexOf('/');
while (slashIndex >= 0) { while (slashIndex >= 0) {
writeDirectoryEntry(zipOutputStream, createdEntries, entryName.substring(0, slashIndex + 1)); writeDirectoryEntry(zipOutputStream, createdEntries, entryName.substring(0, slashIndex + 1), progressState);
slashIndex = entryName.indexOf('/', slashIndex + 1); slashIndex = entryName.indexOf('/', slashIndex + 1);
} }
} }
private void writeDirectoryEntry(ZipOutputStream zipOutputStream, Set<String> createdEntries, String entryName) throws IOException { private void writeDirectoryEntry(ZipOutputStream zipOutputStream,
Set<String> createdEntries,
String entryName,
ArchiveBuildProgressState progressState) throws IOException {
if (!createdEntries.add(entryName)) { if (!createdEntries.add(entryName)) {
return; return;
} }
zipOutputStream.putNextEntry(new ZipEntry(entryName)); zipOutputStream.putNextEntry(new ZipEntry(entryName));
zipOutputStream.closeEntry(); zipOutputStream.closeEntry();
if (progressState != null) {
progressState.processedDirectoryCount += 1;
reportArchiveProgress(progressState);
}
} }
private void writeFileEntry(ZipOutputStream zipOutputStream, Set<String> createdEntries, String entryName, byte[] content) private void writeFileEntry(ZipOutputStream zipOutputStream,
Set<String> createdEntries,
String entryName,
ArchiveBuildProgressState progressState,
byte[] content)
throws IOException { throws IOException {
if (!createdEntries.add(entryName)) { if (!createdEntries.add(entryName)) {
return; return;
@@ -1106,6 +1423,10 @@ public class FileService {
zipOutputStream.putNextEntry(new ZipEntry(entryName)); zipOutputStream.putNextEntry(new ZipEntry(entryName));
zipOutputStream.write(content); zipOutputStream.write(content);
zipOutputStream.closeEntry(); zipOutputStream.closeEntry();
if (progressState != null) {
progressState.processedFileCount += 1;
reportArchiveProgress(progressState);
}
} }
private void recordFileEvent(User user, private void recordFileEvent(User user,
@@ -1170,6 +1491,16 @@ public class FileService {
} }
} }
private void cleanupWrittenBlobs(List<String> writtenBlobObjectKeys, RuntimeException ex) {
for (String objectKey : writtenBlobObjectKeys) {
try {
fileContentStorage.deleteBlob(objectKey);
} catch (RuntimeException cleanupEx) {
ex.addSuppressed(cleanupEx);
}
}
}
private FileBlob createAndSaveBlob(String objectKey, String contentType, long size) { private FileBlob createAndSaveBlob(String objectKey, String contentType, long size) {
FileBlob blob = new FileBlob(); FileBlob blob = new FileBlob();
blob.setObjectKey(objectKey); blob.setObjectKey(objectKey);
@@ -1248,4 +1579,57 @@ public class FileService {
private interface BlobWriteOperation<T> { private interface BlobWriteOperation<T> {
T run(); T run();
} }
public static record ZipCompatibleArchive(List<ZipCompatibleArchiveEntry> entries, String commonRootDirectoryName) {
}
public static record ZipCompatibleArchiveEntry(String relativePath, boolean directory, byte[] content) {
}
public static record ExternalFileImport(String path, String filename, String contentType, byte[] content) {
public long size() {
return content == null ? 0L : content.length;
}
}
public record ArchiveSourceSummary(int fileCount, int directoryCount) {
}
public record ArchiveBuildProgress(int processedFileCount,
int totalFileCount,
int processedDirectoryCount,
int totalDirectoryCount) {
}
@FunctionalInterface
public interface ArchiveBuildProgressListener {
void onProgress(ArchiveBuildProgress progress);
}
public record ExternalImportProgress(int processedFileCount,
int totalFileCount,
int processedDirectoryCount,
int totalDirectoryCount) {
}
@FunctionalInterface
public interface ExternalImportProgressListener {
void onProgress(ExternalImportProgress progress);
}
private static final class ArchiveBuildProgressState {
private final ArchiveBuildProgressListener listener;
private final int totalFileCount;
private final int totalDirectoryCount;
private int processedFileCount;
private int processedDirectoryCount;
private ArchiveBuildProgressState(ArchiveBuildProgressListener listener,
int totalFileCount,
int totalDirectoryCount) {
this.listener = listener;
this.totalFileCount = totalFileCount;
this.totalDirectoryCount = totalDirectoryCount;
}
}
} }

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import java.time.LocalDateTime; import java.time.LocalDateTime;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import jakarta.persistence.Column; import jakarta.persistence.Column;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.events;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.events;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.events;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.events;
public enum FileEventType { public enum FileEventType {
CREATED, CREATED,

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.policy;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.policy;
public record StoragePolicyCapabilities( public record StoragePolicyCapabilities(
boolean directUpload, boolean directUpload,

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.policy;
public enum StoragePolicyCredentialMode { public enum StoragePolicyCredentialMode {
NONE, NONE,

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.policy;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.policy;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.config.FileStorageProperties; import com.yoyuzh.config.FileStorageProperties;
@@ -57,7 +57,7 @@ public class StoragePolicyService implements CommandLineRunner {
policy.setMaxSizeBytes(properties.getMaxFileSize()); policy.setMaxSizeBytes(properties.getMaxFileSize());
policy.setCapabilitiesJson(writeCapabilities(new StoragePolicyCapabilities( policy.setCapabilitiesJson(writeCapabilities(new StoragePolicyCapabilities(
true, true,
false, true,
true, true,
true, true,
false, false,

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.policy;
public enum StoragePolicyType { public enum StoragePolicyType {
LOCAL, LOCAL,

View File

@@ -1,5 +1,6 @@
package com.yoyuzh.files; package com.yoyuzh.files.search;
import com.yoyuzh.files.core.StoredFile;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.FetchType; import jakarta.persistence.FetchType;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.search;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.search;
import java.time.LocalDateTime; import java.time.LocalDateTime;

View File

@@ -1,9 +1,12 @@
package com.yoyuzh.files; package com.yoyuzh.files.search;
import com.yoyuzh.api.v2.ApiV2ErrorCode; import com.yoyuzh.api.v2.ApiV2ErrorCode;
import com.yoyuzh.api.v2.ApiV2Exception; import com.yoyuzh.api.v2.ApiV2Exception;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.core.FileMetadataResponse;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.share;
import java.time.LocalDateTime; import java.time.LocalDateTime;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.share;
import java.time.LocalDateTime; import java.time.LocalDateTime;

View File

@@ -1,6 +1,7 @@
package com.yoyuzh.files; package com.yoyuzh.files.share;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.files.core.StoredFile;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.FetchType; import jakarta.persistence.FetchType;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.share;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.share;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.share;
import com.yoyuzh.api.v2.ApiV2ErrorCode; import com.yoyuzh.api.v2.ApiV2ErrorCode;
import com.yoyuzh.api.v2.ApiV2Exception; import com.yoyuzh.api.v2.ApiV2Exception;
@@ -7,10 +7,15 @@ import com.yoyuzh.api.v2.shares.ImportShareV2Request;
import com.yoyuzh.api.v2.shares.ShareV2Response; import com.yoyuzh.api.v2.shares.ShareV2Response;
import com.yoyuzh.api.v2.shares.VerifySharePasswordV2Request; import com.yoyuzh.api.v2.shares.VerifySharePasswordV2Request;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.files.core.FileMetadataResponse;
import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository;
import jakarta.transaction.Transactional; import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;
import org.springframework.http.ResponseEntity;
import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
@@ -85,6 +90,17 @@ public class ShareV2Service {
return importedFile; return importedFile;
} }
@Transactional
public ResponseEntity<?> downloadSharedFile(String token, String password) {
FileShareLink shareLink = getShareLink(token);
ensureShareNotExpired(shareLink);
ensureDownloadAllowed(shareLink);
ensurePasswordAccepted(shareLink, password);
shareLink.setDownloadCount(shareLink.getDownloadCountOrZero() + 1);
return fileService.download(shareLink.getOwner(), shareLink.getFile().getId());
}
@Transactional @Transactional
public Page<ShareV2Response> listOwnedShares(User user, Pageable pageable) { public Page<ShareV2Response> listOwnedShares(User user, Pageable pageable) {
return fileShareLinkRepository.findByOwnerIdOrderByCreatedAtDesc(user.getId(), pageable) return fileShareLinkRepository.findByOwnerIdOrderByCreatedAtDesc(user.getId(), pageable)
@@ -114,6 +130,18 @@ public class ShareV2Service {
throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "import disabled"); throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "import disabled");
} }
ensureQuotaAvailable(shareLink);
}
private void ensureDownloadAllowed(FileShareLink shareLink) {
if (!shareLink.isAllowDownloadEnabled()) {
throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "download disabled");
}
ensureQuotaAvailable(shareLink);
}
private void ensureQuotaAvailable(FileShareLink shareLink) {
Integer maxDownloads = shareLink.getMaxDownloads(); Integer maxDownloads = shareLink.getMaxDownloads();
if (maxDownloads != null && shareLink.getDownloadCountOrZero() >= maxDownloads) { if (maxDownloads != null && shareLink.getDownloadCountOrZero() >= maxDownloads) {
throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "share quota exceeded"); throw new ApiV2Exception(ApiV2ErrorCode.PERMISSION_DENIED, "share quota exceeded");

View File

@@ -48,6 +48,26 @@ public interface FileContentStorage {
void deleteBlob(String objectKey); void deleteBlob(String objectKey);
default String createMultipartUpload(String objectKey, String contentType) {
throw new UnsupportedOperationException("Multipart upload is not supported by this storage");
}
default PreparedUpload prepareMultipartPartUpload(String objectKey,
String uploadId,
int partNumber,
String contentType,
long size) {
throw new UnsupportedOperationException("Multipart upload is not supported by this storage");
}
default void completeMultipartUpload(String objectKey, String uploadId, java.util.List<MultipartCompletedPart> parts) {
throw new UnsupportedOperationException("Multipart upload is not supported by this storage");
}
default void abortMultipartUpload(String objectKey, String uploadId) {
throw new UnsupportedOperationException("Multipart upload is not supported by this storage");
}
String createBlobDownloadUrl(String objectKey, String filename); String createBlobDownloadUrl(String objectKey, String filename);
void createDirectory(Long userId, String logicalPath); void createDirectory(Long userId, String logicalPath);

View File

@@ -0,0 +1,7 @@
package com.yoyuzh.files.storage;
public record MultipartCompletedPart(
int partNumber,
String etag
) {
}

View File

@@ -9,22 +9,31 @@ import org.springframework.web.multipart.MultipartFile;
import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpMethod;
import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.Duration; import java.time.Duration;
import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -160,6 +169,90 @@ public class S3FileContentStorage implements FileContentStorage {
} }
} }
@Override
public String createMultipartUpload(String objectKey, String contentType) {
S3FileRuntimeSession session = sessionProvider.currentSession();
CreateMultipartUploadRequest.Builder requestBuilder = CreateMultipartUploadRequest.builder()
.bucket(session.bucket())
.key(normalizeObjectKey(objectKey));
if (StringUtils.hasText(contentType)) {
requestBuilder.contentType(contentType);
}
try {
return session.s3Client().createMultipartUpload(requestBuilder.build()).uploadId();
} catch (S3Exception ex) {
throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload init failed");
}
}
@Override
public PreparedUpload prepareMultipartPartUpload(String objectKey,
String uploadId,
int partNumber,
String contentType,
long size) {
S3FileRuntimeSession session = sessionProvider.currentSession();
UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
.bucket(session.bucket())
.key(normalizeObjectKey(objectKey))
.uploadId(uploadId)
.partNumber(partNumber)
.contentLength(size)
.build();
UploadPartPresignRequest presignRequest = UploadPartPresignRequest.builder()
.signatureDuration(Duration.ofSeconds(Math.max(1, properties.getTtlSeconds())))
.uploadPartRequest(uploadPartRequest)
.build();
PresignedUploadPartRequest presignedRequest = session.s3Presigner().presignUploadPart(presignRequest);
Map<String, String> headers = flattenSignedHeaders(presignedRequest.signedHeaders());
if (StringUtils.hasText(contentType)) {
headers.put("Content-Type", contentType);
}
return new PreparedUpload(
true,
presignedRequest.url().toString(),
resolveUploadMethod(presignedRequest),
headers,
objectKey
);
}
@Override
public void completeMultipartUpload(String objectKey, String uploadId, List<MultipartCompletedPart> parts) {
S3FileRuntimeSession session = sessionProvider.currentSession();
List<CompletedPart> completedParts = parts.stream()
.sorted(Comparator.comparingInt(MultipartCompletedPart::partNumber))
.map(part -> CompletedPart.builder()
.partNumber(part.partNumber())
.eTag(part.etag())
.build())
.toList();
try {
session.s3Client().completeMultipartUpload(CompleteMultipartUploadRequest.builder()
.bucket(session.bucket())
.key(normalizeObjectKey(objectKey))
.uploadId(uploadId)
.multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
.build());
} catch (S3Exception ex) {
throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload complete failed");
}
}
@Override
public void abortMultipartUpload(String objectKey, String uploadId) {
S3FileRuntimeSession session = sessionProvider.currentSession();
try {
session.s3Client().abortMultipartUpload(AbortMultipartUploadRequest.builder()
.bucket(session.bucket())
.key(normalizeObjectKey(objectKey))
.uploadId(uploadId)
.build());
} catch (S3Exception ex) {
throw new BusinessException(ErrorCode.UNKNOWN, "Multipart upload abort failed");
}
}
@Override @Override
public String createBlobDownloadUrl(String objectKey, String filename) { public String createBlobDownloadUrl(String objectKey, String filename) {
return createDownloadUrl(sessionProvider.currentSession(), normalizeObjectKey(objectKey), filename); return createDownloadUrl(sessionProvider.currentSession(), normalizeObjectKey(objectKey), filename);
@@ -330,6 +423,13 @@ public class S3FileContentStorage implements FileContentStorage {
return presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : "POST"; return presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : "POST";
} }
private String resolveUploadMethod(PresignedUploadPartRequest presignedRequest) {
if (presignedRequest.httpRequest() == null) {
return "PUT";
}
return presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : presignedRequest.httpRequest().method().name();
}
private Map<String, String> resolveUploadHeaders(PresignedPutObjectRequest presignedRequest, String contentType) { private Map<String, String> resolveUploadHeaders(PresignedPutObjectRequest presignedRequest, String contentType) {
Map<String, String> headers = flattenSignedHeaders(presignedRequest.signedHeaders()); Map<String, String> headers = flattenSignedHeaders(presignedRequest.signedHeaders());
if (StringUtils.hasText(contentType)) { if (StringUtils.hasText(contentType)) {

View File

@@ -0,0 +1,164 @@
package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.files.core.FileMetadataResponse;
import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository;
import jakarta.transaction.Transactional;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.util.LinkedHashMap;
import java.util.Map;
@Component
@Transactional
public class ArchiveBackgroundTaskHandler implements BackgroundTaskHandler {
private final StoredFileRepository storedFileRepository;
private final UserRepository userRepository;
private final FileService fileService;
private final ObjectMapper objectMapper;
public ArchiveBackgroundTaskHandler(StoredFileRepository storedFileRepository,
UserRepository userRepository,
FileService fileService,
ObjectMapper objectMapper) {
this.storedFileRepository = storedFileRepository;
this.userRepository = userRepository;
this.fileService = fileService;
this.objectMapper = objectMapper;
}
@Override
public boolean supports(BackgroundTaskType type) {
return type == BackgroundTaskType.ARCHIVE;
}
@Override
public BackgroundTaskHandlerResult handle(BackgroundTask task) {
return handle(task, publicStatePatch -> {
});
}
@Override
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
Map<String, Object> state = parseState(task.getPrivateStateJson(), task.getPublicStateJson());
Long fileId = extractLong(state.get("fileId"));
String outputPath = extractText(state.get("outputPath"));
String outputFilename = extractText(state.get("outputFilename"));
if (fileId == null) {
throw new IllegalStateException("archive task missing fileId");
}
if (!StringUtils.hasText(outputPath) || !StringUtils.hasText(outputFilename)) {
throw new IllegalStateException("archive task missing output target");
}
StoredFile source = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId())
.orElseThrow(() -> new IllegalStateException("archive task file not found"));
User user = userRepository.findById(task.getUserId())
.orElseThrow(() -> new IllegalStateException("archive task user not found"));
FileService.ArchiveSourceSummary summary = fileService.summarizeArchiveSource(source);
progressReporter.report(progressPatch(0, summary.fileCount(), 0, summary.directoryCount()));
byte[] archiveBytes = fileService.buildArchiveBytes(source, progress ->
progressReporter.report(progressPatch(
progress.processedFileCount(),
progress.totalFileCount(),
progress.processedDirectoryCount(),
progress.totalDirectoryCount()
)));
FileMetadataResponse archivedFile = fileService.importExternalFile(
user,
outputPath,
outputFilename,
"application/zip",
archiveBytes.length,
archiveBytes
);
Map<String, Object> publicStatePatch = new LinkedHashMap<>();
publicStatePatch.put("worker", "archive");
publicStatePatch.put("archivedFileId", archivedFile.id());
publicStatePatch.put("archivedFilename", archivedFile.filename());
publicStatePatch.put("archivedPath", archivedFile.path());
publicStatePatch.put("archiveSize", archiveBytes.length);
publicStatePatch.putAll(progressPatch(
summary.fileCount(),
summary.fileCount(),
summary.directoryCount(),
summary.directoryCount()
));
return new BackgroundTaskHandlerResult(publicStatePatch);
}
private Map<String, Object> progressPatch(int processedFileCount,
int totalFileCount,
int processedDirectoryCount,
int totalDirectoryCount) {
Map<String, Object> patch = new LinkedHashMap<>();
patch.put("processedFileCount", processedFileCount);
patch.put("totalFileCount", totalFileCount);
patch.put("processedDirectoryCount", processedDirectoryCount);
patch.put("totalDirectoryCount", totalDirectoryCount);
patch.put("progressPercent", calculateProgressPercent(
processedFileCount,
totalFileCount,
processedDirectoryCount,
totalDirectoryCount
));
return patch;
}
private int calculateProgressPercent(int processedFileCount,
int totalFileCount,
int processedDirectoryCount,
int totalDirectoryCount) {
int total = Math.max(0, totalFileCount) + Math.max(0, totalDirectoryCount);
int processed = Math.max(0, processedFileCount) + Math.max(0, processedDirectoryCount);
if (total <= 0) {
return 100;
}
return Math.min(100, (int) Math.floor((processed * 100.0d) / total));
}
private Map<String, Object> parseState(String privateStateJson, String publicStateJson) {
Map<String, Object> state = new LinkedHashMap<>(parseJsonObject(publicStateJson));
state.putAll(parseJsonObject(privateStateJson));
return state;
}
private Map<String, Object> parseJsonObject(String json) {
if (!StringUtils.hasText(json)) {
return Map.of();
}
try {
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("archive task state is invalid", ex);
}
}
private Long extractLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text && StringUtils.hasText(text)) {
return Long.parseLong(text.trim());
}
return null;
}
private String extractText(Object value) {
if (value instanceof String text && StringUtils.hasText(text)) {
return text.trim();
}
return null;
}
}

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.tasks;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
@@ -18,6 +18,7 @@ import java.time.LocalDateTime;
@Table(name = "portal_background_task", indexes = { @Table(name = "portal_background_task", indexes = {
@Index(name = "idx_background_task_user_created_at", columnList = "user_id,created_at"), @Index(name = "idx_background_task_user_created_at", columnList = "user_id,created_at"),
@Index(name = "idx_background_task_status_created_at", columnList = "status,created_at"), @Index(name = "idx_background_task_status_created_at", columnList = "status,created_at"),
@Index(name = "idx_background_task_status_lease_expires_at", columnList = "status,lease_expires_at"),
@Index(name = "idx_background_task_correlation_id", columnList = "correlation_id") @Index(name = "idx_background_task_correlation_id", columnList = "correlation_id")
}) })
public class BackgroundTask { public class BackgroundTask {
@@ -49,6 +50,24 @@ public class BackgroundTask {
@Column(name = "error_message", length = 512) @Column(name = "error_message", length = 512)
private String errorMessage; private String errorMessage;
@Column(name = "attempt_count", nullable = false)
private Integer attemptCount;
@Column(name = "max_attempts", nullable = false)
private Integer maxAttempts;
@Column(name = "next_run_at")
private LocalDateTime nextRunAt;
@Column(name = "lease_owner", length = 128)
private String leaseOwner;
@Column(name = "lease_expires_at")
private LocalDateTime leaseExpiresAt;
@Column(name = "heartbeat_at")
private LocalDateTime heartbeatAt;
@Column(name = "created_at", nullable = false) @Column(name = "created_at", nullable = false)
private LocalDateTime createdAt; private LocalDateTime createdAt;
@@ -70,6 +89,12 @@ public class BackgroundTask {
if (status == null) { if (status == null) {
status = BackgroundTaskStatus.QUEUED; status = BackgroundTaskStatus.QUEUED;
} }
if (attemptCount == null) {
attemptCount = 0;
}
if (maxAttempts == null) {
maxAttempts = 1;
}
if (publicStateJson == null) { if (publicStateJson == null) {
publicStateJson = "{}"; publicStateJson = "{}";
} }
@@ -147,6 +172,54 @@ public class BackgroundTask {
this.errorMessage = errorMessage; this.errorMessage = errorMessage;
} }
public Integer getAttemptCount() {
return attemptCount;
}
public void setAttemptCount(Integer attemptCount) {
this.attemptCount = attemptCount;
}
public Integer getMaxAttempts() {
return maxAttempts;
}
public void setMaxAttempts(Integer maxAttempts) {
this.maxAttempts = maxAttempts;
}
public LocalDateTime getNextRunAt() {
return nextRunAt;
}
public void setNextRunAt(LocalDateTime nextRunAt) {
this.nextRunAt = nextRunAt;
}
public String getLeaseOwner() {
return leaseOwner;
}
public void setLeaseOwner(String leaseOwner) {
this.leaseOwner = leaseOwner;
}
public LocalDateTime getLeaseExpiresAt() {
return leaseExpiresAt;
}
public void setLeaseExpiresAt(LocalDateTime leaseExpiresAt) {
this.leaseExpiresAt = leaseExpiresAt;
}
public LocalDateTime getHeartbeatAt() {
return heartbeatAt;
}
public void setHeartbeatAt(LocalDateTime heartbeatAt) {
this.heartbeatAt = heartbeatAt;
}
public LocalDateTime getCreatedAt() { public LocalDateTime getCreatedAt() {
return createdAt; return createdAt;
} }

View File

@@ -0,0 +1,19 @@
package com.yoyuzh.files.tasks;
public enum BackgroundTaskFailureCategory {
UNSUPPORTED_INPUT(false),
DATA_STATE(false),
TRANSIENT_INFRASTRUCTURE(true),
RATE_LIMITED(true),
UNKNOWN(true);
private final boolean retryable;
BackgroundTaskFailureCategory(boolean retryable) {
this.retryable = retryable;
}
public boolean isRetryable() {
return retryable;
}
}

View File

@@ -0,0 +1,12 @@
package com.yoyuzh.files.tasks;
public interface BackgroundTaskHandler {
boolean supports(BackgroundTaskType type);
BackgroundTaskHandlerResult handle(BackgroundTask task);
default BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
return handle(task);
}
}

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.tasks;
import java.util.Map; import java.util.Map;

View File

@@ -0,0 +1,8 @@
package com.yoyuzh.files.tasks;
class BackgroundTaskLeaseLostException extends RuntimeException {
BackgroundTaskLeaseLostException(Long taskId, String workerOwner) {
super("background task lease lost: taskId=" + taskId + ", workerOwner=" + workerOwner);
}
}

View File

@@ -0,0 +1,9 @@
package com.yoyuzh.files.tasks;
import java.util.Map;
@FunctionalInterface
public interface BackgroundTaskProgressReporter {
void report(Map<String, Object> publicStatePatch);
}

View File

@@ -0,0 +1,103 @@
package com.yoyuzh.files.tasks;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
public interface BackgroundTaskRepository extends JpaRepository<BackgroundTask, Long> {
Page<BackgroundTask> findByUserIdOrderByCreatedAtDesc(Long userId, Pageable pageable);
Optional<BackgroundTask> findByIdAndUserId(Long id, Long userId);
List<BackgroundTask> findByStatusOrderByCreatedAtAsc(BackgroundTaskStatus status, Pageable pageable);
List<BackgroundTask> findByStatusOrderByUpdatedAtAsc(BackgroundTaskStatus status);
@Query("""
select task.id from BackgroundTask task
where task.status = :status
and (task.nextRunAt is null or task.nextRunAt <= :now)
order by coalesce(task.nextRunAt, task.createdAt) asc, task.createdAt asc
""")
List<Long> findReadyTaskIdsByStatusOrder(@Param("status") BackgroundTaskStatus status,
@Param("now") LocalDateTime now,
Pageable pageable);
@Modifying
@Query("""
update BackgroundTask task
set task.status = :runningStatus,
task.errorMessage = null,
task.nextRunAt = null,
task.attemptCount = task.attemptCount + 1,
task.leaseOwner = :leaseOwner,
task.leaseExpiresAt = :leaseExpiresAt,
task.heartbeatAt = :heartbeatAt,
task.updatedAt = :updatedAt
where task.id = :id
and task.status = :queuedStatus
""")
int claimQueuedTask(@Param("id") Long id,
@Param("queuedStatus") BackgroundTaskStatus queuedStatus,
@Param("runningStatus") BackgroundTaskStatus runningStatus,
@Param("leaseOwner") String leaseOwner,
@Param("leaseExpiresAt") LocalDateTime leaseExpiresAt,
@Param("heartbeatAt") LocalDateTime heartbeatAt,
@Param("updatedAt") LocalDateTime updatedAt);
@Query("""
select task.id from BackgroundTask task
where task.status = :status
and (task.leaseExpiresAt is null or task.leaseExpiresAt <= :now)
order by coalesce(task.leaseExpiresAt, task.updatedAt, task.createdAt) asc
""")
List<Long> findExpiredRunningTaskIds(@Param("status") BackgroundTaskStatus status,
@Param("now") LocalDateTime now,
Pageable pageable);
@Modifying
@Query("""
update BackgroundTask task
set task.status = :queuedStatus,
task.errorMessage = null,
task.finishedAt = null,
task.nextRunAt = null,
task.leaseOwner = null,
task.leaseExpiresAt = null,
task.heartbeatAt = null,
task.updatedAt = :updatedAt
where task.id = :id
and task.status = :runningStatus
and (task.leaseExpiresAt is null or task.leaseExpiresAt <= :now)
""")
int requeueExpiredRunningTask(@Param("id") Long id,
@Param("runningStatus") BackgroundTaskStatus runningStatus,
@Param("queuedStatus") BackgroundTaskStatus queuedStatus,
@Param("now") LocalDateTime now,
@Param("updatedAt") LocalDateTime updatedAt);
@Modifying
@Query("""
update BackgroundTask task
set task.leaseExpiresAt = :leaseExpiresAt,
task.heartbeatAt = :heartbeatAt,
task.updatedAt = :updatedAt
where task.id = :id
and task.status = :runningStatus
and task.leaseOwner = :leaseOwner
""")
int refreshRunningTaskLease(@Param("id") Long id,
@Param("runningStatus") BackgroundTaskStatus runningStatus,
@Param("leaseOwner") String leaseOwner,
@Param("leaseExpiresAt") LocalDateTime leaseExpiresAt,
@Param("heartbeatAt") LocalDateTime heartbeatAt,
@Param("updatedAt") LocalDateTime updatedAt);
}

View File

@@ -0,0 +1,683 @@
package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.api.v2.ApiV2ErrorCode;
import com.yoyuzh.api.v2.ApiV2Exception;
import com.yoyuzh.auth.User;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
@Service
@RequiredArgsConstructor
public class BackgroundTaskService {
static final String STATE_PHASE_KEY = "phase";
static final String STATE_ATTEMPT_COUNT_KEY = "attemptCount";
static final String STATE_MAX_ATTEMPTS_KEY = "maxAttempts";
static final String STATE_RETRY_SCHEDULED_KEY = "retryScheduled";
static final String STATE_NEXT_RETRY_AT_KEY = "nextRetryAt";
static final String STATE_RETRY_DELAY_SECONDS_KEY = "retryDelaySeconds";
static final String STATE_LAST_FAILURE_MESSAGE_KEY = "lastFailureMessage";
static final String STATE_LAST_FAILURE_AT_KEY = "lastFailureAt";
static final String STATE_FAILURE_CATEGORY_KEY = "failureCategory";
static final String STATE_WORKER_OWNER_KEY = "workerOwner";
static final String STATE_HEARTBEAT_AT_KEY = "heartbeatAt";
static final String STATE_LEASE_EXPIRES_AT_KEY = "leaseExpiresAt";
static final String STATE_STARTED_AT_KEY = "startedAt";
private static final List<String> ZIP_COMPATIBLE_EXTENSIONS = List.of(".zip", ".jar", ".war");
private static final List<String> MEDIA_EXTENSIONS = List.of(
".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg",
".mp4", ".mov", ".mkv", ".webm", ".avi",
".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"
);
private static final List<String> RETRY_TRANSIENT_STATE_KEYS = List.of(
STATE_RETRY_SCHEDULED_KEY,
STATE_NEXT_RETRY_AT_KEY,
STATE_RETRY_DELAY_SECONDS_KEY,
STATE_LAST_FAILURE_MESSAGE_KEY,
STATE_LAST_FAILURE_AT_KEY,
STATE_FAILURE_CATEGORY_KEY
);
private static final List<String> RUNNING_TRANSIENT_STATE_KEYS = List.of(
STATE_WORKER_OWNER_KEY,
STATE_LEASE_EXPIRES_AT_KEY
);
private static final int EXPIRED_RUNNING_TASK_BATCH_SIZE = 100;
private final BackgroundTaskRepository backgroundTaskRepository;
private final StoredFileRepository storedFileRepository;
private final ObjectMapper objectMapper;
@Transactional
public BackgroundTask createQueuedFileTask(User user,
BackgroundTaskType type,
Long fileId,
String requestedPath,
String correlationId) {
StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, user.getId())
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "file not found"));
String logicalPath = buildLogicalPath(file);
if (!logicalPath.equals(normalizeLogicalPath(requestedPath))) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task path does not match file path");
}
validateTaskTarget(type, file);
Map<String, Object> publicState = fileState(file, logicalPath);
Map<String, Object> privateState = new LinkedHashMap<>(publicState);
privateState.put("taskType", type.name());
if (type == BackgroundTaskType.ARCHIVE) {
String outputPath = file.getPath();
String outputFilename = file.getFilename() + ".zip";
publicState.put("outputPath", outputPath);
publicState.put("outputFilename", outputFilename);
privateState.put("outputPath", outputPath);
privateState.put("outputFilename", outputFilename);
} else if (type == BackgroundTaskType.EXTRACT) {
String outputPath = file.getPath();
String outputDirectoryName = deriveExtractOutputDirectoryName(file.getFilename());
publicState.put("outputPath", outputPath);
publicState.put("outputDirectoryName", outputDirectoryName);
privateState.put("outputPath", outputPath);
privateState.put("outputDirectoryName", outputDirectoryName);
}
return createQueuedTask(user, type, publicState, privateState, correlationId);
}
@Transactional
public BackgroundTask createQueuedTask(User user,
BackgroundTaskType type,
Map<String, Object> publicState,
Map<String, Object> privateState,
String correlationId) {
BackgroundTask task = new BackgroundTask();
task.setUserId(user.getId());
task.setType(type);
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setAttemptCount(0);
task.setMaxAttempts(resolveMaxAttempts(type));
task.setNextRunAt(null);
Map<String, Object> nextPublicState = new LinkedHashMap<>(publicState == null ? Map.of() : publicState);
nextPublicState.put(STATE_PHASE_KEY, "queued");
nextPublicState.putAll(retryStatePatch(task.getAttemptCount(), task.getMaxAttempts()));
task.setPublicStateJson(toJson(nextPublicState));
task.setPrivateStateJson(toJson(privateState));
task.setCorrelationId(normalizeCorrelationId(correlationId));
return backgroundTaskRepository.save(task);
}
public Page<BackgroundTask> listOwnedTasks(User user, Pageable pageable) {
return backgroundTaskRepository.findByUserIdOrderByCreatedAtDesc(user.getId(), pageable);
}
public BackgroundTask getOwnedTask(User user, Long id) {
return backgroundTaskRepository.findByIdAndUserId(id, user.getId())
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
}
@Transactional
public BackgroundTask cancelOwnedTask(User user, Long id) {
BackgroundTask task = getOwnedTask(user, id);
if (task.isTerminal()) {
return task;
}
if (task.getStatus() == BackgroundTaskStatus.QUEUED || task.getStatus() == BackgroundTaskStatus.RUNNING) {
task.setStatus(BackgroundTaskStatus.CANCELLED);
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
Map.of(
STATE_PHASE_KEY, "cancelled",
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
),
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
));
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
return task;
}
@Transactional
public BackgroundTask retryOwnedTask(User user, Long id) {
BackgroundTask task = getOwnedTask(user, id);
if (task.getStatus() != BackgroundTaskStatus.FAILED) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "only failed tasks can be retried");
}
task.setAttemptCount(0);
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts()));
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setFinishedAt(null);
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markRunning(User user, Long id) {
BackgroundTask task = getOwnedTask(user, id);
if (task.isTerminal()) {
return task;
}
task.setStatus(BackgroundTaskStatus.RUNNING);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
Map.of(
STATE_PHASE_KEY, "running",
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts()
),
List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY)
));
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markCompleted(User user, Long id) {
BackgroundTask task = getOwnedTask(user, id);
if (task.isTerminal()) {
return task;
}
task.setStatus(BackgroundTaskStatus.COMPLETED);
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
Map.of(
STATE_PHASE_KEY, "completed",
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
),
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
));
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markFailed(User user, Long id, String errorMessage) {
BackgroundTask task = getOwnedTask(user, id);
if (task.isTerminal()) {
return task;
}
task.setStatus(BackgroundTaskStatus.FAILED);
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
Map.of(
STATE_PHASE_KEY, "failed",
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_LAST_FAILURE_MESSAGE_KEY, StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed",
STATE_LAST_FAILURE_AT_KEY, LocalDateTime.now().toString(),
STATE_FAILURE_CATEGORY_KEY, BackgroundTaskFailureCategory.UNKNOWN.name(),
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
),
removableStateKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY), RUNNING_TRANSIENT_STATE_KEYS)
));
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed");
return backgroundTaskRepository.save(task);
}
@Transactional
public int requeueExpiredRunningTasks() {
LocalDateTime now = LocalDateTime.now();
int recovered = 0;
for (Long taskId : backgroundTaskRepository.findExpiredRunningTaskIds(
BackgroundTaskStatus.RUNNING,
now,
PageRequest.of(0, EXPIRED_RUNNING_TASK_BATCH_SIZE)
)) {
int requeued = backgroundTaskRepository.requeueExpiredRunningTask(
taskId,
BackgroundTaskStatus.RUNNING,
BackgroundTaskStatus.QUEUED,
now,
now
);
if (requeued != 1) {
continue;
}
BackgroundTask task = backgroundTaskRepository.findById(taskId)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
resetTaskToQueued(task);
backgroundTaskRepository.save(task);
recovered += 1;
}
return recovered;
}
public List<Long> findQueuedTaskIds(int limit) {
if (limit <= 0) {
return List.of();
}
return backgroundTaskRepository.findReadyTaskIdsByStatusOrder(
BackgroundTaskStatus.QUEUED,
LocalDateTime.now(),
PageRequest.of(0, limit)
);
}
@Transactional
public Optional<BackgroundTask> claimQueuedTask(Long id, String workerOwner, long leaseDurationSeconds) {
LocalDateTime now = LocalDateTime.now();
LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds));
int claimed = backgroundTaskRepository.claimQueuedTask(
id,
BackgroundTaskStatus.QUEUED,
BackgroundTaskStatus.RUNNING,
workerOwner,
leaseExpiresAt,
now,
now
);
if (claimed != 1) {
return Optional.empty();
}
Optional<BackgroundTask> task = backgroundTaskRepository.findById(id);
task.ifPresent(claimedTask -> {
claimedTask.setLeaseOwner(workerOwner);
claimedTask.setLeaseExpiresAt(leaseExpiresAt);
claimedTask.setHeartbeatAt(now);
claimedTask.setPublicStateJson(mergePublicStateJson(
claimedTask.getPublicStateJson(),
runningStatePatch(claimedTask, workerOwner, now, leaseExpiresAt, true),
RETRY_TRANSIENT_STATE_KEYS
));
});
task.ifPresent(backgroundTaskRepository::save);
return task;
}
@Transactional
public BackgroundTask markWorkerTaskProgress(Long id,
String workerOwner,
Map<String, Object> publicStatePatch,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
task.setLeaseOwner(workerOwner);
task.setLeaseExpiresAt(leaseTouch.leaseExpiresAt());
task.setHeartbeatAt(leaseTouch.now());
Map<String, Object> nextPatch = new LinkedHashMap<>(runningStatePatch(
task,
workerOwner,
leaseTouch.now(),
leaseTouch.leaseExpiresAt(),
false
));
if (publicStatePatch != null) {
nextPatch.putAll(publicStatePatch);
}
task.setPublicStateJson(mergePublicStateJson(task.getPublicStateJson(), nextPatch));
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markWorkerTaskCompleted(Long id,
String workerOwner,
Map<String, Object> publicStatePatch,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
Map<String, Object> nextPatch = new LinkedHashMap<>(publicStatePatch == null ? Map.of() : publicStatePatch);
nextPatch.put(STATE_PHASE_KEY, "completed");
nextPatch.put(STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount());
nextPatch.put(STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts());
nextPatch.put(STATE_HEARTBEAT_AT_KEY, leaseTouch.now().toString());
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
nextPatch,
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
));
task.setStatus(BackgroundTaskStatus.COMPLETED);
task.setNextRunAt(null);
clearLease(task);
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markWorkerTaskFailed(Long id,
String workerOwner,
String errorMessage,
BackgroundTaskFailureCategory failureCategory,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
String normalizedErrorMessage = StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed";
LocalDateTime now = leaseTouch.now();
if (failureCategory.isRetryable() && hasRemainingAttempts(task)) {
long retryDelaySeconds = resolveRetryDelaySeconds(task.getType(), failureCategory, task.getAttemptCount());
LocalDateTime nextRunAt = now.plusSeconds(retryDelaySeconds);
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setNextRunAt(nextRunAt);
clearLease(task);
task.setFinishedAt(null);
task.setErrorMessage(null);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
Map.of(
STATE_PHASE_KEY, "queued",
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_RETRY_SCHEDULED_KEY, true,
STATE_NEXT_RETRY_AT_KEY, nextRunAt.toString(),
STATE_RETRY_DELAY_SECONDS_KEY, retryDelaySeconds,
STATE_LAST_FAILURE_MESSAGE_KEY, normalizedErrorMessage,
STATE_LAST_FAILURE_AT_KEY, now.toString(),
STATE_FAILURE_CATEGORY_KEY, failureCategory.name(),
STATE_HEARTBEAT_AT_KEY, now.toString()
),
RUNNING_TRANSIENT_STATE_KEYS
));
return backgroundTaskRepository.save(task);
}
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
Map.of(
STATE_PHASE_KEY, "failed",
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_LAST_FAILURE_MESSAGE_KEY, normalizedErrorMessage,
STATE_LAST_FAILURE_AT_KEY, now.toString(),
STATE_FAILURE_CATEGORY_KEY, failureCategory.name(),
STATE_HEARTBEAT_AT_KEY, now.toString()
),
removableStateKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY, STATE_RETRY_DELAY_SECONDS_KEY), RUNNING_TRANSIENT_STATE_KEYS)
));
task.setStatus(BackgroundTaskStatus.FAILED);
task.setFinishedAt(now);
task.setErrorMessage(normalizedErrorMessage);
return backgroundTaskRepository.save(task);
}
private String normalizeCorrelationId(String correlationId) {
if (StringUtils.hasText(correlationId)) {
return correlationId.trim();
}
return UUID.randomUUID().toString().replace("-", "");
}
private void validateTaskTarget(BackgroundTaskType type, StoredFile file) {
if (type == BackgroundTaskType.ARCHIVE) {
return;
}
if (file.isDirectory()) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task target type is not supported");
}
if (type == BackgroundTaskType.EXTRACT && !isZipCompatibleArchive(file)) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "extract task only supports zip-compatible archives");
}
if (type == BackgroundTaskType.MEDIA_META && !isMediaLike(file)) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "media metadata task only supports media files");
}
}
private Map<String, Object> fileState(StoredFile file, String logicalPath) {
Map<String, Object> state = new LinkedHashMap<>();
state.put("fileId", file.getId());
state.put("path", logicalPath);
state.put("filename", file.getFilename());
state.put("directory", file.isDirectory());
state.put("contentType", file.getContentType());
state.put("size", file.getSize());
return state;
}
private boolean isZipCompatibleArchive(StoredFile file) {
String contentType = normalizeContentType(file.getContentType());
if (contentType.contains("zip") || contentType.contains("java-archive")) {
return true;
}
return hasExtension(file.getFilename(), ZIP_COMPATIBLE_EXTENSIONS);
}
private boolean isMediaLike(StoredFile file) {
String contentType = normalizeContentType(file.getContentType());
if (contentType.startsWith("image/") || contentType.startsWith("video/") || contentType.startsWith("audio/")) {
return true;
}
return hasExtension(file.getFilename(), MEDIA_EXTENSIONS);
}
private String deriveExtractOutputDirectoryName(String filename) {
if (!StringUtils.hasText(filename)) {
return "extracted";
}
String trimmed = filename.trim();
String lower = trimmed.toLowerCase(Locale.ROOT);
for (String extension : ZIP_COMPATIBLE_EXTENSIONS) {
if (lower.endsWith(extension) && trimmed.length() > extension.length()) {
return trimmed.substring(0, trimmed.length() - extension.length());
}
}
int lastDot = trimmed.lastIndexOf('.');
if (lastDot > 0) {
return trimmed.substring(0, lastDot);
}
return trimmed;
}
private boolean hasExtension(String filename, List<String> extensions) {
if (!StringUtils.hasText(filename)) {
return false;
}
String normalized = filename.toLowerCase(Locale.ROOT);
return extensions.stream().anyMatch(normalized::endsWith);
}
private String normalizeContentType(String contentType) {
if (!StringUtils.hasText(contentType)) {
return "";
}
return contentType.trim().toLowerCase(Locale.ROOT);
}
private String buildLogicalPath(StoredFile file) {
String parent = normalizeLogicalPath(file.getPath());
if ("/".equals(parent)) {
return "/" + file.getFilename();
}
return parent + "/" + file.getFilename();
}
private String normalizeLogicalPath(String path) {
if (!StringUtils.hasText(path)) {
return "/";
}
String normalized = path.trim().replace('\\', '/');
while (normalized.contains("//")) {
normalized = normalized.replace("//", "/");
}
if (!normalized.startsWith("/")) {
normalized = "/" + normalized;
}
while (normalized.length() > 1 && normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
private String toJson(Map<String, Object> value) {
Map<String, Object> safeValue = value == null ? new LinkedHashMap<>() : new LinkedHashMap<>(value);
try {
return objectMapper.writeValueAsString(safeValue);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to serialize background task state", ex);
}
}
private Map<String, Object> parseJsonObject(String value) {
if (!StringUtils.hasText(value)) {
return new LinkedHashMap<>();
}
try {
return objectMapper.readValue(value, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to parse background task state", ex);
}
}
private String mergePublicStateJson(String currentValue, Map<String, Object> patch) {
return mergePublicStateJson(currentValue, patch, List.of());
}
private String mergePublicStateJson(String currentValue, Map<String, Object> patch, List<String> keysToRemove) {
Map<String, Object> nextPublicState = parseJsonObject(currentValue);
if (keysToRemove != null) {
keysToRemove.forEach(nextPublicState::remove);
}
if (patch != null) {
nextPublicState.putAll(patch);
}
return toJson(nextPublicState);
}
private String resetPublicStateForRetry(String privateStateJson, int attemptCount, int maxAttempts) {
Map<String, Object> nextPublicState = parseJsonObject(privateStateJson);
nextPublicState.remove("taskType");
nextPublicState.put(STATE_PHASE_KEY, "queued");
nextPublicState.putAll(retryStatePatch(attemptCount, maxAttempts));
return toJson(nextPublicState);
}
private void resetTaskToQueued(BackgroundTask task) {
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts()));
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setFinishedAt(null);
task.setErrorMessage(null);
}
private int resolveMaxAttempts(BackgroundTaskType type) {
return switch (type) {
case ARCHIVE -> 4;
case EXTRACT -> 3;
case MEDIA_META -> 2;
default -> 1;
};
}
private Map<String, Object> retryStatePatch(int attemptCount, int maxAttempts) {
Map<String, Object> patch = new LinkedHashMap<>();
patch.put(STATE_ATTEMPT_COUNT_KEY, attemptCount);
patch.put(STATE_MAX_ATTEMPTS_KEY, maxAttempts);
return patch;
}
private boolean hasRemainingAttempts(BackgroundTask task) {
return task.getAttemptCount() != null
&& task.getMaxAttempts() != null
&& task.getAttemptCount() < task.getMaxAttempts();
}
private long resolveRetryDelaySeconds(BackgroundTaskType type,
BackgroundTaskFailureCategory failureCategory,
Integer attemptCount) {
int safeAttemptCount = attemptCount == null ? 1 : Math.max(1, attemptCount);
long baseDelaySeconds = switch (type) {
case ARCHIVE -> 30L;
case EXTRACT -> 45L;
case MEDIA_META -> 15L;
default -> 30L;
};
if (failureCategory == BackgroundTaskFailureCategory.RATE_LIMITED) {
baseDelaySeconds *= 4L;
} else if (failureCategory == BackgroundTaskFailureCategory.UNKNOWN) {
baseDelaySeconds *= 2L;
}
long delay = baseDelaySeconds * (1L << Math.min(safeAttemptCount - 1, 2));
return Math.min(delay, baseDelaySeconds * 4L);
}
private LeaseTouch refreshLease(Long id, String workerOwner, long leaseDurationSeconds) {
LocalDateTime now = LocalDateTime.now();
LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds));
int refreshed = backgroundTaskRepository.refreshRunningTaskLease(
id,
BackgroundTaskStatus.RUNNING,
workerOwner,
leaseExpiresAt,
now,
now
);
if (refreshed != 1) {
throw new BackgroundTaskLeaseLostException(id, workerOwner);
}
return new LeaseTouch(now, leaseExpiresAt);
}
private Map<String, Object> runningStatePatch(BackgroundTask task,
String workerOwner,
LocalDateTime heartbeatAt,
LocalDateTime leaseExpiresAt,
boolean includeStartedAt) {
Map<String, Object> patch = new LinkedHashMap<>();
patch.put(STATE_PHASE_KEY, "running");
patch.put(STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount());
patch.put(STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts());
patch.put(STATE_WORKER_OWNER_KEY, workerOwner);
patch.put(STATE_HEARTBEAT_AT_KEY, heartbeatAt.toString());
patch.put(STATE_LEASE_EXPIRES_AT_KEY, leaseExpiresAt.toString());
if (includeStartedAt) {
patch.put(STATE_STARTED_AT_KEY, heartbeatAt.toString());
}
return patch;
}
private List<String> removableStateKeys(List<String> primary, List<String> secondary) {
List<String> keys = new java.util.ArrayList<>(primary);
keys.addAll(secondary);
return keys;
}
private void clearLease(BackgroundTask task) {
task.setLeaseOwner(null);
task.setLeaseExpiresAt(null);
task.setHeartbeatAt(null);
}
private record LeaseTouch(LocalDateTime now, LocalDateTime leaseExpiresAt) {
}
}

View File

@@ -0,0 +1,23 @@
package com.yoyuzh.files.tasks;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
@Component
@RequiredArgsConstructor
@Slf4j
public class BackgroundTaskStartupRecovery {
private final BackgroundTaskService backgroundTaskService;
@EventListener(ApplicationReadyEvent.class)
public void recoverOnStartup() {
int recovered = backgroundTaskService.requeueExpiredRunningTasks();
if (recovered > 0) {
log.warn("Recovered {} expired RUNNING background task leases back to QUEUED on startup", recovered);
}
}
}

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.tasks;
public enum BackgroundTaskStatus { public enum BackgroundTaskStatus {
QUEUED, QUEUED,

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.tasks;
public enum BackgroundTaskType { public enum BackgroundTaskType {
ARCHIVE, ARCHIVE,

View File

@@ -0,0 +1,167 @@
package com.yoyuzh.files.tasks;
import com.yoyuzh.common.BusinessException;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
@Component
public class BackgroundTaskWorker {
private static final int DEFAULT_BATCH_SIZE = 5;
private static final long DEFAULT_LEASE_DURATION_SECONDS = 120L;
private final BackgroundTaskService backgroundTaskService;
private final List<BackgroundTaskHandler> handlers;
private final String workerOwner;
public BackgroundTaskWorker(BackgroundTaskService backgroundTaskService,
List<BackgroundTaskHandler> handlers) {
this.backgroundTaskService = backgroundTaskService;
this.handlers = List.copyOf(handlers);
this.workerOwner = UUID.randomUUID().toString().replace("-", "");
}
@Scheduled(
fixedDelayString = "${app.background-tasks.worker.fixed-delay-ms:30000}",
initialDelayString = "${app.background-tasks.worker.initial-delay-ms:30000}"
)
public void runScheduledBatch() {
processQueuedTasks(DEFAULT_BATCH_SIZE);
}
public int processQueuedTasks(int maxTasks) {
backgroundTaskService.requeueExpiredRunningTasks();
int processedCount = 0;
for (Long taskId : backgroundTaskService.findQueuedTaskIds(maxTasks)) {
var claimedTask = backgroundTaskService.claimQueuedTask(taskId, workerOwner, DEFAULT_LEASE_DURATION_SECONDS);
if (claimedTask.isEmpty()) {
continue;
}
execute(claimedTask.get());
processedCount += 1;
}
return processedCount;
}
private void execute(BackgroundTask task) {
try {
backgroundTaskService.markWorkerTaskProgress(
task.getId(),
workerOwner,
Map.of(BackgroundTaskService.STATE_PHASE_KEY, resolveRunningPhase(task.getType())),
DEFAULT_LEASE_DURATION_SECONDS
);
BackgroundTaskHandler handler = findHandler(task);
BackgroundTaskHandlerResult result = handler.handle(task, publicStatePatch ->
backgroundTaskService.markWorkerTaskProgress(
task.getId(),
workerOwner,
publicStatePatch,
DEFAULT_LEASE_DURATION_SECONDS
));
backgroundTaskService.markWorkerTaskCompleted(
task.getId(),
workerOwner,
result.publicStatePatch(),
DEFAULT_LEASE_DURATION_SECONDS
);
} catch (BackgroundTaskLeaseLostException ignored) {
// Another worker reclaimed the task after this worker stopped heartbeating.
} catch (Exception ex) {
String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage();
try {
backgroundTaskService.markWorkerTaskFailed(
task.getId(),
workerOwner,
message,
classifyFailure(ex),
DEFAULT_LEASE_DURATION_SECONDS
);
} catch (BackgroundTaskLeaseLostException ignored) {
// Lease already moved to another worker; keep current worker from overwriting state.
}
}
}
private String resolveRunningPhase(BackgroundTaskType type) {
return switch (type) {
case ARCHIVE -> "archiving";
case EXTRACT -> "extracting";
case MEDIA_META -> "extracting-metadata";
default -> "running";
};
}
private BackgroundTaskHandler findHandler(BackgroundTask task) {
return handlers.stream()
.filter(handler -> handler.supports(task.getType()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("No background task handler for " + task.getType()));
}
private BackgroundTaskFailureCategory classifyFailure(Throwable throwable) {
if (containsRateLimitSignal(throwable)) {
return BackgroundTaskFailureCategory.RATE_LIMITED;
}
Throwable current = throwable;
while (current != null) {
if (current instanceof BusinessException || current instanceof IllegalArgumentException) {
return BackgroundTaskFailureCategory.UNSUPPORTED_INPUT;
}
if (current instanceof IllegalStateException) {
return BackgroundTaskFailureCategory.DATA_STATE;
}
if (current instanceof SocketTimeoutException
|| current instanceof TimeoutException
|| current instanceof ConnectException
|| current instanceof IOException
|| current instanceof UncheckedIOException) {
return BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE;
}
current = current.getCause();
}
if (containsTransientInfrastructureSignal(throwable)) {
return BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE;
}
return BackgroundTaskFailureCategory.UNKNOWN;
}
private boolean containsRateLimitSignal(Throwable throwable) {
String message = collectMessages(throwable);
return message.contains("429")
|| message.contains("too many requests")
|| message.contains("rate limit")
|| message.contains("throttle");
}
private boolean containsTransientInfrastructureSignal(Throwable throwable) {
String message = collectMessages(throwable);
return message.contains("timeout")
|| message.contains("temporarily unavailable")
|| message.contains("connection reset")
|| message.contains("broken pipe")
|| message.contains("connection refused");
}
private String collectMessages(Throwable throwable) {
StringBuilder builder = new StringBuilder();
Throwable current = throwable;
while (current != null) {
if (current.getMessage() != null) {
builder.append(current.getMessage().toLowerCase()).append(' ');
}
current = current.getCause();
}
return builder.toString();
}
}

View File

@@ -0,0 +1,356 @@
package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository;
import jakarta.transaction.Transactional;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@Component
@Transactional
public class ExtractBackgroundTaskHandler implements BackgroundTaskHandler {
private final StoredFileRepository storedFileRepository;
private final UserRepository userRepository;
private final FileService fileService;
private final ObjectMapper objectMapper;
public ExtractBackgroundTaskHandler(StoredFileRepository storedFileRepository,
UserRepository userRepository,
FileService fileService,
ObjectMapper objectMapper) {
this.storedFileRepository = storedFileRepository;
this.userRepository = userRepository;
this.fileService = fileService;
this.objectMapper = objectMapper;
}
@Override
public boolean supports(BackgroundTaskType type) {
return type == BackgroundTaskType.EXTRACT;
}
@Override
public BackgroundTaskHandlerResult handle(BackgroundTask task) {
return handle(task, publicStatePatch -> {
});
}
@Override
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
Map<String, Object> state = parseState(task.getPrivateStateJson(), task.getPublicStateJson());
Long fileId = extractLong(state.get("fileId"));
String outputPath = extractText(state.get("outputPath"));
String outputDirectoryName = extractText(state.get("outputDirectoryName"));
if (fileId == null) {
throw new IllegalStateException("extract task missing fileId");
}
if (!StringUtils.hasText(outputPath) || !StringUtils.hasText(outputDirectoryName)) {
throw new IllegalStateException("extract task missing output target");
}
StoredFile archive = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId())
.orElseThrow(() -> new IllegalStateException("extract task file not found"));
if (archive.isDirectory()) {
throw new IllegalStateException("extract task only supports files");
}
User user = userRepository.findById(task.getUserId())
.orElseThrow(() -> new IllegalStateException("extract task user not found"));
ExtractPlan plan = parseArchivePlan(archive, outputPath, outputDirectoryName);
progressReporter.report(progressPatch(0, plan.files().size(), 0, plan.directories().size()));
executePlan(user, plan, progressReporter);
Map<String, Object> publicStatePatch = new LinkedHashMap<>();
publicStatePatch.put("worker", "extract");
publicStatePatch.put("extractedPath", plan.extractedPath());
publicStatePatch.put("extractedFileCount", plan.files().size());
publicStatePatch.put("extractedDirectoryCount", plan.directories().size());
publicStatePatch.putAll(progressPatch(
plan.files().size(),
plan.files().size(),
plan.directories().size(),
plan.directories().size()
));
return new BackgroundTaskHandlerResult(publicStatePatch);
}
private void executePlan(User user, ExtractPlan plan, BackgroundTaskProgressReporter progressReporter) {
fileService.importExternalFilesAtomically(
user,
plan.directories(),
plan.files().stream()
.map(file -> new FileService.ExternalFileImport(
file.parentPath(),
file.filename(),
file.contentType(),
file.content()
))
.toList(),
progress -> progressReporter.report(progressPatch(
progress.processedFileCount(),
progress.totalFileCount(),
progress.processedDirectoryCount(),
progress.totalDirectoryCount()
))
);
}
private Map<String, Object> progressPatch(int processedFileCount,
int totalFileCount,
int processedDirectoryCount,
int totalDirectoryCount) {
Map<String, Object> patch = new LinkedHashMap<>();
patch.put("processedFileCount", processedFileCount);
patch.put("totalFileCount", totalFileCount);
patch.put("processedDirectoryCount", processedDirectoryCount);
patch.put("totalDirectoryCount", totalDirectoryCount);
patch.put("progressPercent", calculateProgressPercent(
processedFileCount,
totalFileCount,
processedDirectoryCount,
totalDirectoryCount
));
return patch;
}
private int calculateProgressPercent(int processedFileCount,
int totalFileCount,
int processedDirectoryCount,
int totalDirectoryCount) {
int total = Math.max(0, totalFileCount) + Math.max(0, totalDirectoryCount);
int processed = Math.max(0, processedFileCount) + Math.max(0, processedDirectoryCount);
if (total <= 0) {
return 100;
}
return Math.min(100, (int) Math.floor((processed * 100.0d) / total));
}
private ExtractPlan parseArchivePlan(StoredFile archive, String outputPath, String outputDirectoryName) {
FileService.ZipCompatibleArchive zipArchive;
try {
zipArchive = fileService.readZipCompatibleArchive(archive);
} catch (BusinessException ex) {
throw new IllegalStateException("extract task only supports zip-compatible archives", ex);
}
List<ZipItem> items = zipArchive.entries().stream()
.map(entry -> toZipItem(entry, zipArchive.commonRootDirectoryName()))
.filter(item -> StringUtils.hasText(item.path()))
.toList();
if (items.isEmpty()) {
throw new IllegalStateException("extract task archive is empty");
}
String normalizedOutputPath = normalizeDirectoryPath(outputPath);
if (shouldExtractSingleFileToParent(items, outputDirectoryName)) {
ZipItem fileItem = items.get(0);
return new ExtractPlan(
List.of(),
List.of(new ExtractedFile(
normalizedOutputPath,
outputDirectoryName,
fileItem.contentType(),
fileItem.content()
)),
normalizedOutputPath
);
}
String rootPath = joinPath(normalizedOutputPath, outputDirectoryName);
LinkedHashSet<String> directories = new LinkedHashSet<>();
directories.add(rootPath);
List<ExtractedFile> files = new ArrayList<>();
for (ZipItem item : items) {
if (item.directory()) {
directories.add(joinPath(rootPath, trimTrailingSlash(item.path())));
continue;
}
String relativeParent = extractParentPath(item.path());
String targetParent = StringUtils.hasText(relativeParent) ? joinPath(rootPath, relativeParent) : rootPath;
collectParentDirectories(directories, rootPath, relativeParent);
files.add(new ExtractedFile(
targetParent,
extractLeafName(item.path()),
item.contentType(),
item.content()
));
}
return new ExtractPlan(List.copyOf(directories), List.copyOf(files), rootPath);
}
private void collectParentDirectories(LinkedHashSet<String> directories, String rootPath, String relativeParent) {
if (!StringUtils.hasText(relativeParent)) {
return;
}
String current = "";
for (String segment : relativeParent.split("/")) {
current = StringUtils.hasText(current) ? current + "/" + segment : segment;
directories.add(joinPath(rootPath, current));
}
}
private boolean shouldExtractSingleFileToParent(List<ZipItem> items, String outputDirectoryName) {
if (items.size() != 1) {
return false;
}
ZipItem item = items.get(0);
return !item.directory()
&& !item.path().contains("/")
&& outputDirectoryName.equals(item.path());
}
private ZipItem toZipItem(FileService.ZipCompatibleArchiveEntry entry, String commonRootDirectoryName) {
String path = stripCommonRootDirectory(entry.relativePath(), commonRootDirectoryName);
return new ZipItem(path, entry.directory(), entry.content(), guessContentType(path));
}
private String stripCommonRootDirectory(String relativePath, String commonRootDirectoryName) {
if (!StringUtils.hasText(relativePath) || !StringUtils.hasText(commonRootDirectoryName)) {
return relativePath;
}
String prefix = commonRootDirectoryName + "/";
if (relativePath.equals(commonRootDirectoryName)) {
return "";
}
if (relativePath.startsWith(prefix)) {
return relativePath.substring(prefix.length());
}
return relativePath;
}
private Map<String, Object> parseState(String privateStateJson, String publicStateJson) {
Map<String, Object> state = new LinkedHashMap<>(parseJsonObject(publicStateJson));
state.putAll(parseJsonObject(privateStateJson));
return state;
}
private Map<String, Object> parseJsonObject(String json) {
if (!StringUtils.hasText(json)) {
return Map.of();
}
try {
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("extract task state is invalid", ex);
}
}
private Long extractLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text && StringUtils.hasText(text)) {
return Long.parseLong(text.trim());
}
return null;
}
private String extractText(Object value) {
if (value instanceof String text && StringUtils.hasText(text)) {
return text.trim();
}
return null;
}
private String normalizeDirectoryPath(String path) {
if (!StringUtils.hasText(path)) {
return "/";
}
String normalized = path.trim().replace('\\', '/');
while (normalized.contains("//")) {
normalized = normalized.replace("//", "/");
}
if (!normalized.startsWith("/")) {
normalized = "/" + normalized;
}
while (normalized.length() > 1 && normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
private String joinPath(String parent, String leaf) {
String normalizedParent = normalizeDirectoryPath(parent);
String normalizedLeaf = trimSlashes(leaf);
if (!StringUtils.hasText(normalizedLeaf)) {
return normalizedParent;
}
if ("/".equals(normalizedParent)) {
return "/" + normalizedLeaf;
}
return normalizedParent + "/" + normalizedLeaf;
}
private String trimSlashes(String value) {
String normalized = value == null ? "" : value.trim().replace('\\', '/');
while (normalized.startsWith("/")) {
normalized = normalized.substring(1);
}
while (normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
private String trimTrailingSlash(String value) {
if (value == null) {
return "";
}
String normalized = value;
while (normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
private String extractParentPath(String path) {
int separator = path.lastIndexOf('/');
if (separator < 0) {
return "";
}
return path.substring(0, separator);
}
private String extractLeafName(String path) {
int separator = path.lastIndexOf('/');
if (separator < 0) {
return path;
}
return path.substring(separator + 1);
}
private String guessContentType(String entryPath) {
String contentType = URLConnection.guessContentTypeFromName(entryPath);
if (StringUtils.hasText(contentType)) {
return contentType;
}
return "application/octet-stream";
}
private record ZipItem(String path, boolean directory, byte[] content, String contentType) {
}
private record ExtractedFile(String parentPath, String filename, String contentType, byte[] content) {
}
private record ExtractPlan(List<String> directories, List<ExtractedFile> files, String extractedPath) {
}
}

View File

@@ -1,8 +1,13 @@
package com.yoyuzh.files; package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.files.core.FileBlob;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.search.FileMetadata;
import com.yoyuzh.files.search.FileMetadataRepository;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import jakarta.transaction.Transactional; import jakarta.transaction.Transactional;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -48,6 +53,12 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
@Override @Override
public BackgroundTaskHandlerResult handle(BackgroundTask task) { public BackgroundTaskHandlerResult handle(BackgroundTask task) {
return handle(task, publicStatePatch -> {
});
}
@Override
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
Long fileId = readFileId(task); Long fileId = readFileId(task);
StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId()) StoredFile file = storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, task.getUserId())
.orElseThrow(() -> new IllegalStateException("media metadata task file not found")); .orElseThrow(() -> new IllegalStateException("media metadata task file not found"));
@@ -62,6 +73,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
String contentType = firstText(file.getContentType(), blob.getContentType()); String contentType = firstText(file.getContentType(), blob.getContentType());
long size = firstLong(file.getSize(), blob.getSize()); long size = firstLong(file.getSize(), blob.getSize());
progressReporter.report(Map.of("metadataStage", "loading-content"));
byte[] content = Optional.ofNullable(fileContentStorage.readBlob(blob.getObjectKey())) byte[] content = Optional.ofNullable(fileContentStorage.readBlob(blob.getObjectKey()))
.orElseThrow(() -> new IllegalStateException("media metadata task requires blob content")); .orElseThrow(() -> new IllegalStateException("media metadata task requires blob content"));
@@ -75,6 +87,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
upsertMetadata(file, MEDIA_SIZE, String.valueOf(size)); upsertMetadata(file, MEDIA_SIZE, String.valueOf(size));
try { try {
progressReporter.report(Map.of("metadataStage", "reading-image"));
BufferedImage image = ImageIO.read(new ByteArrayInputStream(content)); BufferedImage image = ImageIO.read(new ByteArrayInputStream(content));
if (image != null) { if (image != null) {
upsertMetadata(file, MEDIA_WIDTH, String.valueOf(image.getWidth())); upsertMetadata(file, MEDIA_WIDTH, String.valueOf(image.getWidth()));
@@ -86,6 +99,7 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
throw new IllegalStateException("media metadata task failed to read image dimensions", ex); throw new IllegalStateException("media metadata task failed to read image dimensions", ex);
} }
publicStatePatch.put("metadataStage", "completed");
return new BackgroundTaskHandlerResult(publicStatePatch); return new BackgroundTaskHandlerResult(publicStatePatch);
} }

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.tasks;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -9,10 +9,7 @@ import java.util.Set;
@Component @Component
public class NoopBackgroundTaskHandler implements BackgroundTaskHandler { public class NoopBackgroundTaskHandler implements BackgroundTaskHandler {
private static final Set<BackgroundTaskType> SUPPORTED_TYPES = Set.of( private static final Set<BackgroundTaskType> SUPPORTED_TYPES = Set.of();
BackgroundTaskType.ARCHIVE,
BackgroundTaskType.EXTRACT
);
@Override @Override
public boolean supports(BackgroundTaskType type) { public boolean supports(BackgroundTaskType type) {

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
import jakarta.validation.constraints.Min; import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
import jakarta.validation.constraints.Min; import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
import java.util.Map; import java.util.Map;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import jakarta.persistence.Column; import jakarta.persistence.Column;
@@ -55,6 +55,9 @@ public class UploadSession {
@Column(name = "object_key", nullable = false, length = 512) @Column(name = "object_key", nullable = false, length = 512)
private String objectKey; private String objectKey;
@Column(name = "multipart_upload_id", length = 255)
private String multipartUploadId;
@Column(name = "storage_policy_id") @Column(name = "storage_policy_id")
private Long storagePolicyId; private Long storagePolicyId;
@@ -163,6 +166,14 @@ public class UploadSession {
this.objectKey = objectKey; this.objectKey = objectKey;
} }
public String getMultipartUploadId() {
return multipartUploadId;
}
public void setMultipartUploadId(String multipartUploadId) {
this.multipartUploadId = multipartUploadId;
}
public Long getStoragePolicyId() { public Long getStoragePolicyId() {
return storagePolicyId; return storagePolicyId;
} }

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
public record UploadSessionCreateCommand( public record UploadSessionCreateCommand(
String path, String path,

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
public record UploadSessionPartCommand( public record UploadSessionPartCommand(
String etag, String etag,

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@@ -6,7 +6,13 @@ import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.config.FileStorageProperties; import com.yoyuzh.config.FileStorageProperties;
import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyService;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import com.yoyuzh.files.storage.MultipartCompletedPart;
import com.yoyuzh.files.storage.PreparedUpload;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -82,7 +88,8 @@ public class UploadSessionService {
session.setContentType(command.contentType()); session.setContentType(command.contentType());
session.setSize(command.size()); session.setSize(command.size());
session.setObjectKey(createBlobObjectKey()); session.setObjectKey(createBlobObjectKey());
session.setStoragePolicyId(storagePolicyService.ensureDefaultPolicy().getId()); StoragePolicy policy = storagePolicyService.ensureDefaultPolicy();
session.setStoragePolicyId(policy.getId());
session.setChunkSize(DEFAULT_CHUNK_SIZE); session.setChunkSize(DEFAULT_CHUNK_SIZE);
session.setChunkCount(calculateChunkCount(command.size(), DEFAULT_CHUNK_SIZE)); session.setChunkCount(calculateChunkCount(command.size(), DEFAULT_CHUNK_SIZE));
session.setUploadedPartsJson("[]"); session.setUploadedPartsJson("[]");
@@ -91,6 +98,9 @@ public class UploadSessionService {
session.setCreatedAt(now); session.setCreatedAt(now);
session.setUpdatedAt(now); session.setUpdatedAt(now);
session.setExpiresAt(now.plusHours(SESSION_TTL_HOURS)); session.setExpiresAt(now.plusHours(SESSION_TTL_HOURS));
if (storagePolicyService.readCapabilities(policy).multipartUpload()) {
session.setMultipartUploadId(fileContentStorage.createMultipartUpload(session.getObjectKey(), session.getContentType()));
}
return uploadSessionRepository.save(session); return uploadSessionRepository.save(session);
} }
@@ -111,6 +121,26 @@ public class UploadSessionService {
return uploadSessionRepository.save(session); return uploadSessionRepository.save(session);
} }
@Transactional(readOnly = true)
public PreparedUpload prepareOwnedPartUpload(User user, String sessionId, int partIndex) {
UploadSession session = getOwnedSession(user, sessionId);
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone());
ensureSessionCanReceivePart(session, now);
if (!StringUtils.hasText(session.getMultipartUploadId())) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话未启用 multipart");
}
if (partIndex < 0 || partIndex >= session.getChunkCount()) {
throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法");
}
return fileContentStorage.prepareMultipartPartUpload(
session.getObjectKey(),
session.getMultipartUploadId(),
partIndex + 1,
session.getContentType(),
resolveChunkSize(session, partIndex)
);
}
@Transactional @Transactional
public UploadSession recordUploadedPart(User user, public UploadSession recordUploadedPart(User user,
String sessionId, String sessionId,
@@ -164,6 +194,13 @@ public class UploadSessionService {
uploadSessionRepository.save(session); uploadSessionRepository.save(session);
try { try {
if (StringUtils.hasText(session.getMultipartUploadId())) {
fileContentStorage.completeMultipartUpload(
session.getObjectKey(),
session.getMultipartUploadId(),
toCompletedParts(session)
);
}
fileService.completeUpload(user, new CompleteUploadRequest( fileService.completeUpload(user, new CompleteUploadRequest(
session.getTargetPath(), session.getTargetPath(),
session.getFilename(), session.getFilename(),
@@ -192,7 +229,11 @@ public class UploadSessionService {
); );
for (UploadSession session : expiredSessions) { for (UploadSession session : expiredSessions) {
try { try {
fileContentStorage.deleteBlob(session.getObjectKey()); if (StringUtils.hasText(session.getMultipartUploadId())) {
fileContentStorage.abortMultipartUpload(session.getObjectKey(), session.getMultipartUploadId());
} else {
fileContentStorage.deleteBlob(session.getObjectKey());
}
} catch (RuntimeException ignored) { } catch (RuntimeException ignored) {
// Expiration is authoritative in the database even if remote object cleanup fails. // Expiration is authoritative in the database even if remote object cleanup fails.
} }
@@ -253,6 +294,41 @@ public class UploadSessionService {
} }
} }
private List<MultipartCompletedPart> toCompletedParts(UploadSession session) {
List<UploadedPart> uploadedParts = readUploadedParts(session).stream()
.sorted(Comparator.comparingInt(UploadedPart::partIndex))
.toList();
if (uploadedParts.size() != session.getChunkCount()) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片不完整");
}
for (int expectedIndex = 0; expectedIndex < session.getChunkCount(); expectedIndex++) {
UploadedPart part = uploadedParts.get(expectedIndex);
if (part.partIndex() != expectedIndex) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片不完整");
}
if (!StringUtils.hasText(part.etag())) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片标识缺失");
}
if (part.size() <= 0 || part.size() > resolveChunkSize(session, expectedIndex)) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片大小不合法");
}
}
return uploadedParts.stream()
.map(part -> new MultipartCompletedPart(part.partIndex() + 1, part.etag()))
.toList();
}
private long resolveChunkSize(UploadSession session, int partIndex) {
if (partIndex < 0 || partIndex >= session.getChunkCount()) {
throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法");
}
if (partIndex < session.getChunkCount() - 1) {
return session.getChunkSize();
}
long remaining = session.getSize() - session.getChunkSize() * (session.getChunkCount() - 1L);
return remaining > 0 ? remaining : session.getChunkSize();
}
private record UploadedPart(int partIndex, String etag, long size, String uploadedAt) { private record UploadedPart(int partIndex, String etag, long size, String uploadedAt) {
} }

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.upload;
public enum UploadSessionStatus { public enum UploadSessionStatus {
CREATED, CREATED,

View File

@@ -5,8 +5,8 @@ import com.yoyuzh.auth.User;
import com.yoyuzh.common.ApiResponse; import com.yoyuzh.common.ApiResponse;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.FileMetadataResponse; import com.yoyuzh.files.core.FileMetadataResponse;
import com.yoyuzh.files.ImportSharedFileRequest; import com.yoyuzh.files.share.ImportSharedFileRequest;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;

View File

@@ -5,8 +5,8 @@ import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.config.FileStorageProperties; import com.yoyuzh.config.FileStorageProperties;
import com.yoyuzh.files.FileMetadataResponse; import com.yoyuzh.files.core.FileMetadataResponse;
import com.yoyuzh.files.FileService; import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;

View File

@@ -4,10 +4,10 @@ import com.yoyuzh.PortalBackendApplication;
import com.yoyuzh.admin.AdminMetricsStateRepository; import com.yoyuzh.admin.AdminMetricsStateRepository;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository; import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.files.FileBlob; import com.yoyuzh.files.core.FileBlob;
import com.yoyuzh.files.FileBlobRepository; import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.StoredFile; import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.transfer.OfflineTransferSessionRepository; import com.yoyuzh.transfer.OfflineTransferSessionRepository;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;

View File

@@ -8,12 +8,12 @@ import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.auth.UserRole; import com.yoyuzh.auth.UserRole;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.FileBlobRepository; import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.FileService; import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.StoredFile; import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.StoragePolicyRepository; import com.yoyuzh.files.policy.StoragePolicyRepository;
import com.yoyuzh.files.StoragePolicyService; import com.yoyuzh.files.policy.StoragePolicyService;
import com.yoyuzh.transfer.OfflineTransferSessionRepository; import com.yoyuzh.transfer.OfflineTransferSessionRepository;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;

View File

@@ -4,9 +4,9 @@ import com.yoyuzh.api.v2.ApiV2ExceptionHandler;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.FileMetadataResponse; import com.yoyuzh.files.core.FileMetadataResponse;
import com.yoyuzh.files.FileSearchQuery; import com.yoyuzh.files.search.FileSearchQuery;
import com.yoyuzh.files.FileSearchService; import com.yoyuzh.files.search.FileSearchService;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.core.MethodParameter; import org.springframework.core.MethodParameter;

View File

@@ -2,9 +2,9 @@ package com.yoyuzh.api.v2.files;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.files.UploadSession; import com.yoyuzh.files.upload.UploadSession;
import com.yoyuzh.files.UploadSessionService; import com.yoyuzh.files.upload.UploadSessionService;
import com.yoyuzh.files.UploadSessionStatus; import com.yoyuzh.files.upload.UploadSessionStatus;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
@@ -19,6 +19,7 @@ import org.springframework.web.method.support.HandlerMethodArgumentResolver;
import org.springframework.web.method.support.ModelAndViewContainer; import org.springframework.web.method.support.ModelAndViewContainer;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
@@ -69,6 +70,7 @@ class UploadSessionV2ControllerTest {
.andExpect(jsonPath("$.data.sessionId").value("session-1")) .andExpect(jsonPath("$.data.sessionId").value("session-1"))
.andExpect(jsonPath("$.data.objectKey").value("blobs/session-1")) .andExpect(jsonPath("$.data.objectKey").value("blobs/session-1"))
.andExpect(jsonPath("$.data.status").value("CREATED")) .andExpect(jsonPath("$.data.status").value("CREATED"))
.andExpect(jsonPath("$.data.multipartUpload").value(true))
.andExpect(jsonPath("$.data.chunkSize").value(8388608)) .andExpect(jsonPath("$.data.chunkSize").value(8388608))
.andExpect(jsonPath("$.data.chunkCount").value(3)); .andExpect(jsonPath("$.data.chunkCount").value(3));
} }
@@ -85,7 +87,8 @@ class UploadSessionV2ControllerTest {
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.code").value(0)) .andExpect(jsonPath("$.code").value(0))
.andExpect(jsonPath("$.data.sessionId").value("session-1")) .andExpect(jsonPath("$.data.sessionId").value("session-1"))
.andExpect(jsonPath("$.data.status").value("CREATED")); .andExpect(jsonPath("$.data.status").value("CREATED"))
.andExpect(jsonPath("$.data.multipartUpload").value(true));
} }
@Test @Test
@@ -127,6 +130,29 @@ class UploadSessionV2ControllerTest {
.andExpect(jsonPath("$.data.status").value("UPLOADING")); .andExpect(jsonPath("$.data.status").value("UPLOADING"));
} }
@Test
void shouldPrepareMultipartPartUploadWithV2Envelope() throws Exception {
User user = createUser(7L);
when(userDetailsService.loadDomainUser("alice")).thenReturn(user);
when(uploadSessionService.prepareOwnedPartUpload(user, "session-1", 1))
.thenReturn(new com.yoyuzh.files.storage.PreparedUpload(
true,
"https://upload.example.com/session-1/part-2",
"PUT",
Map.of("Content-Type", "video/mp4"),
"blobs/session-1"
));
mockMvc.perform(get("/api/v2/files/upload-sessions/session-1/parts/1/prepare")
.with(user(userDetails())))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value(0))
.andExpect(jsonPath("$.data.direct").value(true))
.andExpect(jsonPath("$.data.uploadUrl").value("https://upload.example.com/session-1/part-2"))
.andExpect(jsonPath("$.data.method").value("PUT"))
.andExpect(jsonPath("$.data.headers['Content-Type']").value("video/mp4"));
}
private UserDetails userDetails() { private UserDetails userDetails() {
return org.springframework.security.core.userdetails.User return org.springframework.security.core.userdetails.User
.withUsername("alice") .withUsername("alice")
@@ -172,6 +198,7 @@ class UploadSessionV2ControllerTest {
session.setContentType("video/mp4"); session.setContentType("video/mp4");
session.setSize(20L * 1024 * 1024); session.setSize(20L * 1024 * 1024);
session.setObjectKey("blobs/session-1"); session.setObjectKey("blobs/session-1");
session.setMultipartUploadId("upload-123");
session.setChunkSize(8L * 1024 * 1024); session.setChunkSize(8L * 1024 * 1024);
session.setChunkCount(3); session.setChunkCount(3);
session.setStatus(UploadSessionStatus.CREATED); session.setStatus(UploadSessionStatus.CREATED);

View File

@@ -4,12 +4,12 @@ import com.jayway.jsonpath.JsonPath;
import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.PortalBackendApplication;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository; import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.files.FileBlob; import com.yoyuzh.files.core.FileBlob;
import com.yoyuzh.files.FileBlobRepository; import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.FileShareLink; import com.yoyuzh.files.share.FileShareLink;
import com.yoyuzh.files.FileShareLinkRepository; import com.yoyuzh.files.share.FileShareLinkRepository;
import com.yoyuzh.files.StoredFile; import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -30,6 +30,8 @@ import static org.springframework.security.test.web.servlet.request.SecurityMock
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@@ -274,6 +276,120 @@ class ShareV2ControllerIntegrationTest {
.andExpect(jsonPath("$.code").value(2404)); .andExpect(jsonPath("$.code").value(2404));
} }
@Test
void shouldDownloadSharedFileAndCountQuota() throws Exception {
String createResponse = mockMvc.perform(post("/api/v2/shares")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"maxDownloads": 1,
"allowDownload": true
}
""".formatted(sharedFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
String token = JsonPath.read(createResponse, "$.data.token");
mockMvc.perform(get("/api/v2/shares/{token}", token)
.with(anonymous())
.param("download", "1"))
.andExpect(status().isOk())
.andExpect(header().string("Content-Disposition", "attachment; filename*=UTF-8''notes.txt"))
.andExpect(content().contentType("text/plain"))
.andExpect(content().bytes("hello".getBytes(StandardCharsets.UTF_8)));
mockMvc.perform(get("/api/v2/shares/{token}", token)
.with(anonymous())
.param("download", "1"))
.andExpect(status().isForbidden())
.andExpect(jsonPath("$.code").value(2403));
mockMvc.perform(get("/api/v2/shares/{token}", token).with(anonymous()))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.downloadCount").value(1));
}
@Test
void shouldRejectDownloadWhenSharePolicyBlocksIt() throws Exception {
String passwordResponse = mockMvc.perform(post("/api/v2/shares")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"password": "Share123",
"allowDownload": true
}
""".formatted(sharedFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
String passwordToken = JsonPath.read(passwordResponse, "$.data.token");
mockMvc.perform(get("/api/v2/shares/{token}", passwordToken)
.with(anonymous())
.param("download", "1"))
.andExpect(status().isBadRequest())
.andExpect(jsonPath("$.code").value(2400));
mockMvc.perform(get("/api/v2/shares/{token}", passwordToken)
.with(anonymous())
.param("download", "1")
.param("password", "Share123"))
.andExpect(status().isOk())
.andExpect(content().bytes("hello".getBytes(StandardCharsets.UTF_8)));
String disabledResponse = mockMvc.perform(post("/api/v2/shares")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"allowDownload": false
}
""".formatted(sharedFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
String disabledToken = JsonPath.read(disabledResponse, "$.data.token");
mockMvc.perform(get("/api/v2/shares/{token}", disabledToken)
.with(anonymous())
.param("download", "1"))
.andExpect(status().isForbidden())
.andExpect(jsonPath("$.code").value(2403));
String expiredResponse = mockMvc.perform(post("/api/v2/shares")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"allowDownload": true
}
""".formatted(sharedFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
String expiredToken = JsonPath.read(expiredResponse, "$.data.token");
FileShareLink expiredShare = fileShareLinkRepository.findByToken(expiredToken).orElseThrow();
expiredShare.setExpiresAt(LocalDateTime.now().minusMinutes(1));
fileShareLinkRepository.save(expiredShare);
mockMvc.perform(get("/api/v2/shares/{token}", expiredToken)
.with(anonymous())
.param("download", "1"))
.andExpect(status().isNotFound())
.andExpect(jsonPath("$.code").value(2404));
}
@Test @Test
void shouldDenyDeletingOtherUsersShare() throws Exception { void shouldDenyDeletingOtherUsersShare() throws Exception {
String createResponse = mockMvc.perform(post("/api/v2/shares") String createResponse = mockMvc.perform(post("/api/v2/shares")

View File

@@ -1,16 +1,22 @@
package com.yoyuzh.api.v2.tasks; package com.yoyuzh.api.v2.tasks;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.JsonPath;
import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.PortalBackendApplication;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository; import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.files.BackgroundTask; import com.yoyuzh.files.tasks.BackgroundTask;
import com.yoyuzh.files.BackgroundTaskRepository; import com.yoyuzh.files.tasks.BackgroundTaskRepository;
import com.yoyuzh.files.BackgroundTaskStatus; import com.yoyuzh.files.tasks.BackgroundTaskStatus;
import com.yoyuzh.files.BackgroundTaskType; import com.yoyuzh.files.tasks.BackgroundTaskStartupRecovery;
import com.yoyuzh.files.StoredFile; import com.yoyuzh.files.tasks.BackgroundTaskType;
import com.yoyuzh.files.StoredFileRepository; import com.yoyuzh.files.tasks.BackgroundTaskWorker;
import com.yoyuzh.files.core.FileBlob;
import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.storage.FileContentStorage;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -19,10 +25,17 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MockMvc;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.not;
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.anonymous; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.anonymous;
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
@@ -55,23 +68,39 @@ class BackgroundTaskV2ControllerIntegrationTest {
@Autowired @Autowired
private BackgroundTaskRepository backgroundTaskRepository; private BackgroundTaskRepository backgroundTaskRepository;
@Autowired
private BackgroundTaskWorker backgroundTaskWorker;
@Autowired
private BackgroundTaskStartupRecovery backgroundTaskStartupRecovery;
@Autowired
private FileBlobRepository fileBlobRepository;
@Autowired
private FileContentStorage fileContentStorage;
@Autowired @Autowired
private StoredFileRepository storedFileRepository; private StoredFileRepository storedFileRepository;
@Autowired @Autowired
private ObjectMapper objectMapper; private ObjectMapper objectMapper;
private Long aliceId;
private Long archiveDirectoryId; private Long archiveDirectoryId;
private Long archiveFileId; private Long archiveFileId;
private Long extractFileId; private Long extractFileId;
private Long invalidExtractFileId;
private Long unsupportedExtractFileId;
private Long mediaFileId; private Long mediaFileId;
private Long foreignFileId; private Long foreignFileId;
private Long deletedFileId; private Long deletedFileId;
@BeforeEach @BeforeEach
void setUp() { void setUp() throws Exception {
backgroundTaskRepository.deleteAll(); backgroundTaskRepository.deleteAll();
storedFileRepository.deleteAll(); storedFileRepository.deleteAll();
fileBlobRepository.deleteAll();
userRepository.deleteAll(); userRepository.deleteAll();
User alice = new User(); User alice = new User();
@@ -80,7 +109,8 @@ class BackgroundTaskV2ControllerIntegrationTest {
alice.setPhoneNumber("13800138000"); alice.setPhoneNumber("13800138000");
alice.setPasswordHash("encoded-password"); alice.setPasswordHash("encoded-password");
alice.setCreatedAt(LocalDateTime.now()); alice.setCreatedAt(LocalDateTime.now());
userRepository.save(alice); alice = userRepository.save(alice);
aliceId = alice.getId();
User bob = new User(); User bob = new User();
bob.setUsername("bob"); bob.setUsername("bob");
@@ -91,11 +121,62 @@ class BackgroundTaskV2ControllerIntegrationTest {
bob = userRepository.save(bob); bob = userRepository.save(bob);
archiveDirectoryId = storedFileRepository.save(createFile(alice, "/docs", "archive", true, null, 0L, null)).getId(); archiveDirectoryId = storedFileRepository.save(createFile(alice, "/docs", "archive", true, null, 0L, null)).getId();
archiveFileId = storedFileRepository.save(createFile(alice, "/docs", "archive-source.txt", false, "text/plain", 12L, null)).getId(); storedFileRepository.save(createBlobBackedFile(
extractFileId = storedFileRepository.save(createFile(alice, "/docs", "extract.zip", false, "application/zip", 32L, null)).getId(); alice,
"/docs/archive",
"nested.txt",
"text/plain",
"archive-nested",
"nested-content".getBytes(StandardCharsets.UTF_8)
));
archiveFileId = storedFileRepository.save(createBlobBackedFile(
alice,
"/docs",
"archive-source.txt",
"text/plain",
"archive-source",
"archive-source".getBytes(StandardCharsets.UTF_8)
)).getId();
extractFileId = storedFileRepository.save(createBlobBackedFile(
alice,
"/docs",
"extract.zip",
"application/zip",
"extract-source",
createZipArchive(Map.of(
"extract/", "",
"extract/nested/", "",
"extract/notes.txt", "hello",
"extract/nested/todo.txt", "world"
))
)).getId();
invalidExtractFileId = storedFileRepository.save(createBlobBackedFile(
alice,
"/docs",
"broken.zip",
"application/zip",
"broken-extract",
"not-a-zip".getBytes(StandardCharsets.UTF_8)
)).getId();
unsupportedExtractFileId = storedFileRepository.save(createFile(alice, "/docs", "backup.7z", false, "application/x-7z-compressed", 64L, null)).getId();
mediaFileId = storedFileRepository.save(createFile(alice, "/docs", "media.png", false, "image/png", 24L, null)).getId(); mediaFileId = storedFileRepository.save(createFile(alice, "/docs", "media.png", false, "image/png", 24L, null)).getId();
foreignFileId = storedFileRepository.save(createFile(bob, "/docs", "foreign.zip", false, "application/zip", 32L, null)).getId(); foreignFileId = storedFileRepository.save(createBlobBackedFile(
deletedFileId = storedFileRepository.save(createFile(alice, "/docs", "deleted.zip", false, "application/zip", 32L, LocalDateTime.now())).getId(); bob,
"/docs",
"foreign.zip",
"application/zip",
"foreign-zip",
createZipArchive(Map.of("foreign.txt", "blocked"))
)).getId();
deletedFileId = storedFileRepository.save(createBlobBackedFile(
alice,
"/docs",
"deleted.zip",
"application/zip",
"deleted-zip",
createZipArchive(Map.of("deleted.txt", "gone")),
LocalDateTime.now()
)).getId();
} }
@Test @Test
@@ -132,6 +213,9 @@ class BackgroundTaskV2ControllerIntegrationTest {
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"fileId\":" + archiveDirectoryId))) .andExpect(jsonPath("$.data.publicStateJson", containsString("\"fileId\":" + archiveDirectoryId)))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"path\":\"/docs/archive\""))) .andExpect(jsonPath("$.data.publicStateJson", containsString("\"path\":\"/docs/archive\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"directory\":true"))) .andExpect(jsonPath("$.data.publicStateJson", containsString("\"directory\":true")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":4")))
.andReturn() .andReturn()
.getResponse() .getResponse()
.getContentAsString(); .getContentAsString();
@@ -148,6 +232,11 @@ class BackgroundTaskV2ControllerIntegrationTest {
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.data.type").value("EXTRACT")) .andExpect(jsonPath("$.data.type").value("EXTRACT"))
.andExpect(jsonPath("$.data.status").value("QUEUED")) .andExpect(jsonPath("$.data.status").value("QUEUED"))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputPath\":\"/docs\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputDirectoryName\":\"extract\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
.andReturn() .andReturn()
.getResponse() .getResponse()
.getContentAsString(); .getContentAsString();
@@ -164,6 +253,9 @@ class BackgroundTaskV2ControllerIntegrationTest {
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.data.type").value("MEDIA_META")) .andExpect(jsonPath("$.data.type").value("MEDIA_META"))
.andExpect(jsonPath("$.data.status").value("QUEUED")) .andExpect(jsonPath("$.data.status").value("QUEUED"))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":2")))
.andReturn() .andReturn()
.getResponse() .getResponse()
.getContentAsString(); .getContentAsString();
@@ -187,11 +279,13 @@ class BackgroundTaskV2ControllerIntegrationTest {
mockMvc.perform(delete("/api/v2/tasks/{id}", extractId).with(user("alice"))) mockMvc.perform(delete("/api/v2/tasks/{id}", extractId).with(user("alice")))
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.data.status").value("CANCELLED")); .andExpect(jsonPath("$.data.status").value("CANCELLED"))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"cancelled\"")));
BackgroundTask cancelled = backgroundTaskRepository.findById(extractId).orElseThrow(); BackgroundTask cancelled = backgroundTaskRepository.findById(extractId).orElseThrow();
assertThat(cancelled.getStatus()).isEqualTo(BackgroundTaskStatus.CANCELLED); assertThat(cancelled.getStatus()).isEqualTo(BackgroundTaskStatus.CANCELLED);
assertThat(cancelled.getFinishedAt()).isNotNull(); assertThat(cancelled.getFinishedAt()).isNotNull();
assertThat(cancelled.getPublicStateJson()).contains("\"phase\":\"cancelled\"");
} }
@Test @Test
@@ -216,6 +310,289 @@ class BackgroundTaskV2ControllerIntegrationTest {
mockMvc.perform(delete("/api/v2/tasks/{id}", taskId).with(user("bob"))) mockMvc.perform(delete("/api/v2/tasks/{id}", taskId).with(user("bob")))
.andExpect(status().isNotFound()); .andExpect(status().isNotFound());
mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("bob")))
.andExpect(status().isNotFound());
}
@Test
void shouldRejectExtractTaskForNonZipCompatibleArchive() throws Exception {
mockMvc.perform(post("/api/v2/tasks/extract")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"path": "/docs/backup.7z"
}
""".formatted(unsupportedExtractFileId)))
.andExpect(status().isBadRequest());
}
@Test
void shouldCompleteArchiveTaskThroughWorkerAndExposeTerminalState() throws Exception {
String response = mockMvc.perform(post("/api/v2/tasks/archive")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"path": "/docs/archive-source.txt"
}
""".formatted(archiveFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
assertThat(processedCount).isEqualTo(1);
String taskResponse = mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.status").value("COMPLETED"))
.andReturn()
.getResponse()
.getContentAsString();
Map<String, Object> publicState = readPublicState(taskResponse);
assertThat(publicState).containsEntry("worker", "archive");
assertThat(publicState).containsEntry("archivedFilename", "archive-source.txt.zip");
assertThat(publicState).containsEntry("archivedPath", "/docs");
assertThat(publicState).containsEntry("phase", "completed");
assertThat(publicState).containsEntry("attemptCount", 1);
assertThat(publicState).containsEntry("maxAttempts", 4);
assertThat(publicState).containsEntry("processedFileCount", 1);
assertThat(publicState).containsEntry("totalFileCount", 1);
assertThat(publicState).containsEntry("processedDirectoryCount", 0);
assertThat(publicState).containsEntry("totalDirectoryCount", 0);
assertThat(publicState).containsEntry("progressPercent", 100);
assertThat(publicState.get("heartbeatAt")).isNotNull();
assertThat(publicState).doesNotContainKey("workerOwner");
assertThat(publicState).doesNotContainKey("leaseExpiresAt");
assertThat(publicState.get("archivedFileId")).isNotNull();
assertThat(publicState.get("archiveSize")).isNotNull();
BackgroundTask completed = backgroundTaskRepository.findById(taskId).orElseThrow();
assertThat(completed.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
assertThat(completed.getFinishedAt()).isNotNull();
assertThat(completed.getErrorMessage()).isNull();
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs", "archive-source.txt.zip")).isPresent();
}
@Test
void shouldCompleteExtractTaskThroughWorkerAndExposeTerminalState() throws Exception {
String response = mockMvc.perform(post("/api/v2/tasks/extract")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"path": "/docs/extract.zip"
}
""".formatted(extractFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
assertThat(processedCount).isEqualTo(1);
String taskResponse = mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.status").value("COMPLETED"))
.andReturn()
.getResponse()
.getContentAsString();
Map<String, Object> publicState = readPublicState(taskResponse);
assertThat(publicState).containsEntry("worker", "extract");
assertThat(publicState).containsEntry("extractedPath", "/docs/extract");
assertThat(publicState).containsEntry("extractedFileCount", 2);
assertThat(publicState).containsEntry("extractedDirectoryCount", 2);
assertThat(publicState).containsEntry("phase", "completed");
assertThat(publicState).containsEntry("attemptCount", 1);
assertThat(publicState).containsEntry("maxAttempts", 3);
assertThat(publicState).containsEntry("processedFileCount", 2);
assertThat(publicState).containsEntry("totalFileCount", 2);
assertThat(publicState).containsEntry("processedDirectoryCount", 2);
assertThat(publicState).containsEntry("totalDirectoryCount", 2);
assertThat(publicState).containsEntry("progressPercent", 100);
assertThat(publicState.get("heartbeatAt")).isNotNull();
assertThat(publicState).doesNotContainKey("workerOwner");
assertThat(publicState).doesNotContainKey("leaseExpiresAt");
BackgroundTask completed = backgroundTaskRepository.findById(taskId).orElseThrow();
assertThat(completed.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
assertThat(completed.getFinishedAt()).isNotNull();
assertThat(completed.getErrorMessage()).isNull();
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs/extract", "notes.txt")).isPresent();
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs/extract/nested", "todo.txt")).isPresent();
}
@Test
void shouldMarkExtractTaskFailedWhenWorkerHitsInvalidArchiveContent() throws Exception {
String response = mockMvc.perform(post("/api/v2/tasks/extract")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"path": "/docs/broken.zip"
}
""".formatted(invalidExtractFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
assertThat(processedCount).isEqualTo(1);
mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.status").value("FAILED"))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"failed\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":1")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"failureCategory\":\"DATA_STATE\"")))
.andExpect(jsonPath("$.data.errorMessage").value("extract task only supports zip-compatible archives"));
BackgroundTask failed = backgroundTaskRepository.findById(taskId).orElseThrow();
assertThat(failed.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED);
assertThat(failed.getFinishedAt()).isNotNull();
assertThat(failed.getErrorMessage()).isEqualTo("extract task only supports zip-compatible archives");
}
@Test
void shouldRetryFailedTaskAndResetStateToQueued() throws Exception {
String response = mockMvc.perform(post("/api/v2/tasks/extract")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"path": "/docs/broken.zip"
}
""".formatted(invalidExtractFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
backgroundTaskWorker.processQueuedTasks(5);
String retryResponse = mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("alice")))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.status").value("QUEUED"))
.andExpect(jsonPath("$.data.errorMessage").doesNotExist())
.andExpect(jsonPath("$.data.finishedAt").doesNotExist())
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
.andReturn()
.getResponse()
.getContentAsString();
Map<String, Object> publicState = readPublicState(retryResponse);
assertThat(publicState).containsEntry("phase", "queued");
assertThat(publicState).containsEntry("outputPath", "/docs");
assertThat(publicState).containsEntry("outputDirectoryName", "broken");
assertThat(publicState).containsEntry("attemptCount", 0);
assertThat(publicState).containsEntry("maxAttempts", 3);
assertThat(publicState).doesNotContainKey("worker");
assertThat(publicState).doesNotContainKey("processedFileCount");
assertThat(publicState).doesNotContainKey("totalFileCount");
BackgroundTask retried = backgroundTaskRepository.findById(taskId).orElseThrow();
assertThat(retried.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED);
assertThat(retried.getFinishedAt()).isNull();
assertThat(retried.getErrorMessage()).isNull();
}
@Test
void shouldRejectRetryForNonFailedTask() throws Exception {
String response = mockMvc.perform(post("/api/v2/tasks/archive")
.with(user("alice"))
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"fileId": %d,
"path": "/docs/archive-source.txt"
}
""".formatted(archiveFileId)))
.andExpect(status().isOk())
.andReturn()
.getResponse()
.getContentAsString();
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("alice")))
.andExpect(status().isBadRequest());
}
@Test
void shouldRecoverOnlyExpiredRunningTaskBackToQueuedOnStartup() throws Exception {
BackgroundTask expired = new BackgroundTask();
expired.setType(BackgroundTaskType.EXTRACT);
expired.setStatus(BackgroundTaskStatus.RUNNING);
expired.setUserId(aliceId);
expired.setCorrelationId("recover-1");
expired.setAttemptCount(1);
expired.setMaxAttempts(3);
expired.setLeaseOwner("worker-stale");
expired.setLeaseExpiresAt(LocalDateTime.now().minusMinutes(2));
expired.setHeartbeatAt(LocalDateTime.now().minusMinutes(3));
expired.setPublicStateJson("""
{"fileId":%d,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-stale","attemptCount":1,"maxAttempts":3}
""".formatted(extractFileId));
expired.setPrivateStateJson("""
{"fileId":%d,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"}
""".formatted(extractFileId));
expired.setErrorMessage("stale worker");
expired.setFinishedAt(LocalDateTime.now());
expired = backgroundTaskRepository.save(expired);
BackgroundTask fresh = new BackgroundTask();
fresh.setType(BackgroundTaskType.EXTRACT);
fresh.setStatus(BackgroundTaskStatus.RUNNING);
fresh.setUserId(aliceId);
fresh.setCorrelationId("recover-2");
fresh.setAttemptCount(1);
fresh.setMaxAttempts(3);
fresh.setLeaseOwner("worker-live");
fresh.setLeaseExpiresAt(LocalDateTime.now().plusMinutes(5));
fresh.setHeartbeatAt(LocalDateTime.now());
fresh.setPublicStateJson("""
{"fileId":%d,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-live","attemptCount":1,"maxAttempts":3}
""".formatted(extractFileId));
fresh.setPrivateStateJson("""
{"fileId":%d,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"}
""".formatted(extractFileId));
fresh = backgroundTaskRepository.save(fresh);
backgroundTaskStartupRecovery.recoverOnStartup();
mockMvc.perform(get("/api/v2/tasks/{id}", expired.getId()).with(user("alice")))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.status").value("QUEUED"))
.andExpect(jsonPath("$.data.errorMessage").doesNotExist())
.andExpect(jsonPath("$.data.finishedAt").doesNotExist())
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":1")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputPath\":\"/docs\"")))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputDirectoryName\":\"extract\"")))
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"worker\""))))
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"workerOwner\""))))
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"leaseExpiresAt\""))));
mockMvc.perform(get("/api/v2/tasks/{id}", fresh.getId()).with(user("alice")))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.status").value("RUNNING"))
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"workerOwner\":\"worker-live\"")));
} }
@Test @Test
@@ -274,4 +651,54 @@ class BackgroundTaskV2ControllerIntegrationTest {
file.setDeletedAt(deletedAt); file.setDeletedAt(deletedAt);
return file; return file;
} }
private StoredFile createBlobBackedFile(User user,
String path,
String filename,
String contentType,
String objectKeySuffix,
byte[] content) {
return createBlobBackedFile(user, path, filename, contentType, objectKeySuffix, content, null);
}
private StoredFile createBlobBackedFile(User user,
String path,
String filename,
String contentType,
String objectKeySuffix,
byte[] content,
LocalDateTime deletedAt) {
String objectKey = "blobs/test-background-task/" + objectKeySuffix;
fileContentStorage.storeBlob(objectKey, contentType, content);
FileBlob blob = new FileBlob();
blob.setObjectKey(objectKey);
blob.setContentType(contentType);
blob.setSize((long) content.length);
blob = fileBlobRepository.save(blob);
StoredFile file = createFile(user, path, filename, false, contentType, (long) content.length, deletedAt);
file.setBlob(blob);
return file;
}
private byte[] createZipArchive(Map<String, String> entries) throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) {
for (Map.Entry<String, String> entry : entries.entrySet()) {
zipOutputStream.putNextEntry(new ZipEntry(entry.getKey()));
if (!entry.getKey().endsWith("/")) {
zipOutputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
}
zipOutputStream.closeEntry();
}
}
return outputStream.toByteArray();
}
private Map<String, Object> readPublicState(String taskResponse) throws Exception {
String publicStateJson = JsonPath.read(taskResponse, "$.data.publicStateJson");
return objectMapper.readValue(publicStateJson, new TypeReference<Map<String, Object>>() {
});
}
} }

View File

@@ -7,8 +7,8 @@ import com.yoyuzh.auth.dto.UpdateUserAvatarRequest;
import com.yoyuzh.auth.dto.UpdateUserPasswordRequest; import com.yoyuzh.auth.dto.UpdateUserPasswordRequest;
import com.yoyuzh.auth.dto.UpdateUserProfileRequest; import com.yoyuzh.auth.dto.UpdateUserProfileRequest;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.files.FileService; import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.InitiateUploadResponse; import com.yoyuzh.files.upload.InitiateUploadResponse;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import com.yoyuzh.files.storage.PreparedUpload; import com.yoyuzh.files.storage.PreparedUpload;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;

View File

@@ -39,7 +39,7 @@ class AuthSingleDeviceIntegrationTest {
private UserRepository userRepository; private UserRepository userRepository;
@Autowired @Autowired
private com.yoyuzh.files.StoredFileRepository storedFileRepository; private com.yoyuzh.files.core.StoredFileRepository storedFileRepository;
@Autowired @Autowired
private RefreshTokenRepository refreshTokenRepository; private RefreshTokenRepository refreshTokenRepository;

View File

@@ -1,7 +1,7 @@
package com.yoyuzh.auth; package com.yoyuzh.auth;
import com.yoyuzh.files.FileService; import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;

View File

@@ -1,246 +0,0 @@
package com.yoyuzh.files;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.api.v2.ApiV2Exception;
import com.yoyuzh.auth.User;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class BackgroundTaskServiceTest {
@Mock
private BackgroundTaskRepository backgroundTaskRepository;
@Mock
private StoredFileRepository storedFileRepository;
private BackgroundTaskService backgroundTaskService;
@BeforeEach
void setUp() {
backgroundTaskService = new BackgroundTaskService(backgroundTaskRepository, storedFileRepository, new ObjectMapper());
}
@Test
void shouldRejectTaskCreationForForeignFile() {
User user = createUser(7L);
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(99L, 7L)).thenReturn(Optional.empty());
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
user,
BackgroundTaskType.ARCHIVE,
99L,
"/docs/foreign.txt",
null
)).isInstanceOf(ApiV2Exception.class)
.hasMessage("file not found");
}
@Test
void shouldRejectTaskCreationForDeletedFile() {
User user = createUser(7L);
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(100L, 7L)).thenReturn(Optional.empty());
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
user,
BackgroundTaskType.ARCHIVE,
100L,
"/docs/deleted.txt",
null
)).isInstanceOf(ApiV2Exception.class)
.hasMessage("file not found");
}
@Test
void shouldRejectTaskCreationWhenRequestedPathDoesNotMatchFile() {
User user = createUser(7L);
StoredFile file = createStoredFile(11L, user, "/docs", "real.txt", false, "text/plain", 3L);
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(file));
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
user,
BackgroundTaskType.ARCHIVE,
11L,
"/docs/fake.txt",
null
)).isInstanceOf(ApiV2Exception.class)
.hasMessage("task path does not match file path");
}
@Test
void shouldRejectExtractTaskForDirectory() {
User user = createUser(7L);
StoredFile directory = createStoredFile(12L, user, "/", "bundle", true, null, 0L);
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(12L, 7L)).thenReturn(Optional.of(directory));
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
user,
BackgroundTaskType.EXTRACT,
12L,
"/bundle",
null
)).isInstanceOf(ApiV2Exception.class)
.hasMessage("task target type is not supported");
}
@Test
void shouldRejectMediaMetadataTaskForNonMediaFile() {
User user = createUser(7L);
StoredFile file = createStoredFile(13L, user, "/docs", "notes.txt", false, "text/plain", 9L);
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(13L, 7L)).thenReturn(Optional.of(file));
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
user,
BackgroundTaskType.MEDIA_META,
13L,
"/docs/notes.txt",
null
)).isInstanceOf(ApiV2Exception.class)
.hasMessage("media metadata task only supports media files");
}
@Test
void shouldCreateTaskStateFromServerFilePath() {
User user = createUser(7L);
StoredFile file = createStoredFile(14L, user, "/docs", "photo.png", false, "image/png", 15L);
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(14L, 7L)).thenReturn(Optional.of(file));
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
BackgroundTask task = backgroundTaskService.createQueuedFileTask(
user,
BackgroundTaskType.MEDIA_META,
14L,
"/docs/photo.png",
"media-1"
);
assertThat(task.getPublicStateJson()).contains("\"fileId\":14");
assertThat(task.getPublicStateJson()).contains("\"path\":\"/docs/photo.png\"");
assertThat(task.getPublicStateJson()).contains("\"filename\":\"photo.png\"");
assertThat(task.getPublicStateJson()).contains("\"directory\":false");
assertThat(task.getPublicStateJson()).contains("\"contentType\":\"image/png\"");
assertThat(task.getPublicStateJson()).contains("\"size\":15");
assertThat(task.getPrivateStateJson()).contains("\"taskType\":\"MEDIA_META\"");
}
@Test
void shouldClaimQueuedTaskOnlyWhenRepositoryTransitionSucceeds() {
BackgroundTask task = createTask(1L, BackgroundTaskStatus.RUNNING);
when(backgroundTaskRepository.claimQueuedTask(
eq(1L),
eq(BackgroundTaskStatus.QUEUED),
eq(BackgroundTaskStatus.RUNNING),
any()
)).thenReturn(1);
when(backgroundTaskRepository.findById(1L)).thenReturn(Optional.of(task));
Optional<BackgroundTask> result = backgroundTaskService.claimQueuedTask(1L);
assertThat(result).containsSame(task);
}
@Test
void shouldNotClaimTaskWhenRepositoryTransitionWasSkipped() {
when(backgroundTaskRepository.claimQueuedTask(
eq(2L),
eq(BackgroundTaskStatus.QUEUED),
eq(BackgroundTaskStatus.RUNNING),
any()
)).thenReturn(0);
Optional<BackgroundTask> result = backgroundTaskService.claimQueuedTask(2L);
assertThat(result).isEmpty();
}
@Test
void shouldCompleteRunningWorkerTaskAndMergePublicState() {
BackgroundTask task = createTask(3L, BackgroundTaskStatus.RUNNING);
task.setPublicStateJson("{\"fileId\":11}");
when(backgroundTaskRepository.findById(3L)).thenReturn(Optional.of(task));
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
BackgroundTask result = backgroundTaskService.markWorkerTaskCompleted(3L, Map.of("worker", "noop"));
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
assertThat(result.getFinishedAt()).isNotNull();
assertThat(result.getErrorMessage()).isNull();
assertThat(result.getPublicStateJson()).contains("\"fileId\":11");
assertThat(result.getPublicStateJson()).contains("\"worker\":\"noop\"");
}
@Test
void shouldRecordWorkerFailureMessage() {
BackgroundTask task = createTask(4L, BackgroundTaskStatus.RUNNING);
when(backgroundTaskRepository.findById(4L)).thenReturn(Optional.of(task));
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
BackgroundTask result = backgroundTaskService.markWorkerTaskFailed(4L, "media parser unavailable");
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED);
assertThat(result.getFinishedAt()).isNotNull();
assertThat(result.getErrorMessage()).isEqualTo("media parser unavailable");
}
@Test
void shouldFindQueuedTaskIdsInCreatedOrderLimit() {
BackgroundTask first = createTask(5L, BackgroundTaskStatus.QUEUED);
BackgroundTask second = createTask(6L, BackgroundTaskStatus.QUEUED);
when(backgroundTaskRepository.findByStatusOrderByCreatedAtAsc(eq(BackgroundTaskStatus.QUEUED), any()))
.thenReturn(List.of(first, second));
List<Long> result = backgroundTaskService.findQueuedTaskIds(2);
assertThat(result).containsExactly(5L, 6L);
}
private BackgroundTask createTask(Long id, BackgroundTaskStatus status) {
BackgroundTask task = new BackgroundTask();
task.setId(id);
task.setType(BackgroundTaskType.MEDIA_META);
task.setStatus(status);
task.setUserId(7L);
task.setPublicStateJson("{}");
task.setPrivateStateJson("{}");
return task;
}
private User createUser(Long id) {
User user = new User();
user.setId(id);
user.setUsername("alice");
return user;
}
private StoredFile createStoredFile(Long id,
User user,
String path,
String filename,
boolean directory,
String contentType,
Long size) {
StoredFile file = new StoredFile();
file.setId(id);
file.setUser(user);
file.setPath(path);
file.setFilename(filename);
file.setDirectory(directory);
file.setContentType(contentType);
file.setSize(size);
return file;
}
}

View File

@@ -1,83 +0,0 @@
package com.yoyuzh.files;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class BackgroundTaskWorkerTest {
@Mock
private BackgroundTaskService backgroundTaskService;
@Mock
private BackgroundTaskHandler backgroundTaskHandler;
private BackgroundTaskWorker backgroundTaskWorker;
@BeforeEach
void setUp() {
backgroundTaskWorker = new BackgroundTaskWorker(backgroundTaskService, List.of(backgroundTaskHandler));
}
@Test
void shouldClaimAndCompleteQueuedTaskThroughNoopHandler() {
BackgroundTask task = createTask(1L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L));
when(backgroundTaskService.claimQueuedTask(1L)).thenReturn(Optional.of(task));
when(backgroundTaskHandler.supports(BackgroundTaskType.ARCHIVE)).thenReturn(true);
when(backgroundTaskHandler.handle(task)).thenReturn(new BackgroundTaskHandlerResult(Map.of("worker", "noop")));
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
assertThat(processedCount).isEqualTo(1);
verify(backgroundTaskHandler).handle(task);
verify(backgroundTaskService).markWorkerTaskCompleted(1L, Map.of("worker", "noop"));
}
@Test
void shouldSkipTaskThatWasNotClaimed() {
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L));
when(backgroundTaskService.claimQueuedTask(1L)).thenReturn(Optional.empty());
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
assertThat(processedCount).isZero();
verify(backgroundTaskHandler, never()).handle(org.mockito.ArgumentMatchers.any());
}
@Test
void shouldMarkTaskFailedWhenHandlerThrows() {
BackgroundTask task = createTask(2L, BackgroundTaskType.MEDIA_META, BackgroundTaskStatus.RUNNING);
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(2L));
when(backgroundTaskService.claimQueuedTask(2L)).thenReturn(Optional.of(task));
when(backgroundTaskHandler.supports(BackgroundTaskType.MEDIA_META)).thenReturn(true);
when(backgroundTaskHandler.handle(task)).thenThrow(new IllegalStateException("media parser unavailable"));
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
assertThat(processedCount).isEqualTo(1);
verify(backgroundTaskService).markWorkerTaskFailed(2L, "media parser unavailable");
}
private BackgroundTask createTask(Long id, BackgroundTaskType type, BackgroundTaskStatus status) {
BackgroundTask task = new BackgroundTask();
task.setId(id);
task.setType(type);
task.setStatus(status);
task.setUserId(7L);
task.setPublicStateJson("{}");
task.setPrivateStateJson("{}");
return task;
}
}

View File

@@ -1,4 +1,4 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;

View File

@@ -1,6 +1,10 @@
package com.yoyuzh.files; package com.yoyuzh.files.core;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyCredentialMode;
import com.yoyuzh.files.policy.StoragePolicyService;
import com.yoyuzh.files.policy.StoragePolicyType;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;

Some files were not shown because too many files have changed in this diff Show More