From f59515f5dd109c504a8f3c6b334e457d8e88800a Mon Sep 17 00:00:00 2001 From: yoyuzh Date: Sat, 11 Apr 2026 14:09:31 +0800 Subject: [PATCH] feat(admin): add blob share and task admin apis --- .../com/yoyuzh/admin/AdminController.java | 47 ++ .../yoyuzh/admin/AdminFileBlobResponse.java | 28 + .../java/com/yoyuzh/admin/AdminService.java | 261 ++++++- .../com/yoyuzh/admin/AdminShareResponse.java | 28 + .../com/yoyuzh/admin/AdminTaskLeaseState.java | 7 + .../com/yoyuzh/admin/AdminTaskResponse.java | 32 + .../files/core/FileEntityRepository.java | 29 + .../core/StoredFileEntityRepository.java | 27 + .../files/share/FileShareLinkRepository.java | 31 + .../files/tasks/BackgroundTaskRepository.java | 28 + .../admin/AdminControllerIntegrationTest.java | 209 ++++- .../AdminServiceStoragePolicyCacheTest.java | 310 ++++++++ .../com/yoyuzh/admin/AdminServiceTest.java | 22 +- docs/api-reference.md | 135 ++++ docs/architecture.md | 129 ++++ ...-04-10-cloudreve-gap-next-phase-upgrade.md | 712 ++++++++++++++++++ memory.md | 105 +++ 17 files changed, 2118 insertions(+), 22 deletions(-) create mode 100644 backend/src/main/java/com/yoyuzh/admin/AdminFileBlobResponse.java create mode 100644 backend/src/main/java/com/yoyuzh/admin/AdminShareResponse.java create mode 100644 backend/src/main/java/com/yoyuzh/admin/AdminTaskLeaseState.java create mode 100644 backend/src/main/java/com/yoyuzh/admin/AdminTaskResponse.java create mode 100644 backend/src/test/java/com/yoyuzh/admin/AdminServiceStoragePolicyCacheTest.java create mode 100644 docs/superpowers/plans/2026-04-10-cloudreve-gap-next-phase-upgrade.md diff --git a/backend/src/main/java/com/yoyuzh/admin/AdminController.java b/backend/src/main/java/com/yoyuzh/admin/AdminController.java index 57ed9d6..333e35a 100644 --- a/backend/src/main/java/com/yoyuzh/admin/AdminController.java +++ b/backend/src/main/java/com/yoyuzh/admin/AdminController.java @@ -5,7 +5,11 @@ import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.User; import com.yoyuzh.common.ApiResponse; import com.yoyuzh.common.PageResponse; +import com.yoyuzh.files.core.FileEntityType; import com.yoyuzh.files.tasks.BackgroundTask; +import com.yoyuzh.files.tasks.BackgroundTaskFailureCategory; +import com.yoyuzh.files.tasks.BackgroundTaskStatus; +import com.yoyuzh.files.tasks.BackgroundTaskType; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; @@ -61,6 +65,49 @@ public class AdminController { return ApiResponse.success(adminService.listFiles(page, size, query, ownerQuery)); } + @GetMapping("/file-blobs") + public ApiResponse> fileBlobs(@RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "10") int size, + @RequestParam(defaultValue = "") String userQuery, + @RequestParam(required = false) Long storagePolicyId, + @RequestParam(defaultValue = "") String objectKey, + @RequestParam(required = false) FileEntityType entityType) { + return ApiResponse.success(adminService.listFileBlobs(page, size, userQuery, storagePolicyId, objectKey, entityType)); + } + + @GetMapping("/shares") + public ApiResponse> shares(@RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "10") int size, + @RequestParam(defaultValue = "") String userQuery, + @RequestParam(defaultValue = "") String fileName, + @RequestParam(defaultValue = "") String token, + @RequestParam(required = false) Boolean passwordProtected, + @RequestParam(required = false) Boolean expired) { + return ApiResponse.success(adminService.listShares(page, size, userQuery, fileName, token, passwordProtected, expired)); + } + + @DeleteMapping("/shares/{shareId}") + public ApiResponse deleteShare(@PathVariable Long shareId) { + adminService.deleteShare(shareId); + return ApiResponse.success(); + } + + @GetMapping("/tasks") + public ApiResponse> tasks(@RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "10") int size, + @RequestParam(defaultValue = "") String userQuery, + @RequestParam(required = false) BackgroundTaskType type, + @RequestParam(required = false) BackgroundTaskStatus status, + @RequestParam(required = false) BackgroundTaskFailureCategory failureCategory, + @RequestParam(required = false) AdminTaskLeaseState leaseState) { + return ApiResponse.success(adminService.listTasks(page, size, userQuery, type, status, failureCategory, leaseState)); + } + + @GetMapping("/tasks/{taskId}") + public ApiResponse task(@PathVariable Long taskId) { + return ApiResponse.success(adminService.getTask(taskId)); + } + @GetMapping("/storage-policies") public ApiResponse> storagePolicies() { return ApiResponse.success(adminService.listStoragePolicies()); diff --git a/backend/src/main/java/com/yoyuzh/admin/AdminFileBlobResponse.java b/backend/src/main/java/com/yoyuzh/admin/AdminFileBlobResponse.java new file mode 100644 index 0000000..a57b5a5 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/admin/AdminFileBlobResponse.java @@ -0,0 +1,28 @@ +package com.yoyuzh.admin; + +import com.yoyuzh.files.core.FileEntityType; + +import java.time.LocalDateTime; + +public record AdminFileBlobResponse( + Long entityId, + Long blobId, + String objectKey, + FileEntityType entityType, + Long storagePolicyId, + Long size, + String contentType, + Integer referenceCount, + long linkedStoredFileCount, + long linkedOwnerCount, + String sampleOwnerUsername, + String sampleOwnerEmail, + Long createdByUserId, + String createdByUsername, + LocalDateTime createdAt, + LocalDateTime blobCreatedAt, + boolean blobMissing, + boolean orphanRisk, + boolean referenceMismatch +) { +} diff --git a/backend/src/main/java/com/yoyuzh/admin/AdminService.java b/backend/src/main/java/com/yoyuzh/admin/AdminService.java index 2d702a5..cbfa551 100644 --- a/backend/src/main/java/com/yoyuzh/admin/AdminService.java +++ b/backend/src/main/java/com/yoyuzh/admin/AdminService.java @@ -1,29 +1,42 @@ package com.yoyuzh.admin; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.auth.AuthTokenInvalidationService; import com.yoyuzh.auth.PasswordPolicy; +import com.yoyuzh.auth.RefreshTokenService; import com.yoyuzh.auth.RegistrationInviteService; import com.yoyuzh.auth.User; -import com.yoyuzh.auth.UserRole; import com.yoyuzh.auth.UserRepository; -import com.yoyuzh.auth.RefreshTokenService; +import com.yoyuzh.auth.UserRole; import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.PageResponse; -import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.config.RedisCacheNames; +import com.yoyuzh.files.core.FileEntity; import com.yoyuzh.files.core.FileEntityRepository; import com.yoyuzh.files.core.FileEntityType; import com.yoyuzh.files.core.FileService; import com.yoyuzh.files.core.StoredFile; import com.yoyuzh.files.core.StoredFileEntityRepository; import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.core.FileBlobRepository; import com.yoyuzh.files.policy.StoragePolicy; import com.yoyuzh.files.policy.StoragePolicyRepository; import com.yoyuzh.files.policy.StoragePolicyService; +import com.yoyuzh.files.share.FileShareLink; +import com.yoyuzh.files.share.FileShareLinkRepository; import com.yoyuzh.files.tasks.BackgroundTask; +import com.yoyuzh.files.tasks.BackgroundTaskFailureCategory; +import com.yoyuzh.files.tasks.BackgroundTaskRepository; import com.yoyuzh.files.tasks.BackgroundTaskService; +import com.yoyuzh.files.tasks.BackgroundTaskStatus; import com.yoyuzh.files.tasks.BackgroundTaskType; import com.yoyuzh.transfer.OfflineTransferSessionRepository; import lombok.RequiredArgsConstructor; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; @@ -34,8 +47,12 @@ import org.springframework.util.StringUtils; import java.security.SecureRandom; import java.time.Instant; +import java.time.LocalDateTime; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; @Service @RequiredArgsConstructor @@ -47,6 +64,7 @@ public class AdminService { private final FileService fileService; private final PasswordEncoder passwordEncoder; private final RefreshTokenService refreshTokenService; + private final AuthTokenInvalidationService authTokenInvalidationService; private final RegistrationInviteService registrationInviteService; private final OfflineTransferSessionRepository offlineTransferSessionRepository; private final AdminMetricsService adminMetricsService; @@ -54,7 +72,10 @@ public class AdminService { private final StoragePolicyService storagePolicyService; private final FileEntityRepository fileEntityRepository; private final StoredFileEntityRepository storedFileEntityRepository; + private final BackgroundTaskRepository backgroundTaskRepository; private final BackgroundTaskService backgroundTaskService; + private final FileShareLinkRepository fileShareLinkRepository; + private final ObjectMapper objectMapper; private final SecureRandom secureRandom = new SecureRandom(); public AdminSummaryResponse getSummary() { @@ -98,6 +119,92 @@ public class AdminService { return new PageResponse<>(items, result.getTotalElements(), page, size); } + public PageResponse listFileBlobs(int page, + int size, + String userQuery, + Long storagePolicyId, + String objectKey, + FileEntityType entityType) { + Page result = fileEntityRepository.searchAdminEntities( + normalizeQuery(userQuery), + storagePolicyId, + normalizeQuery(objectKey), + entityType, + PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt")) + ); + List items = result.getContent().stream() + .map(this::toFileBlobResponse) + .toList(); + return new PageResponse<>(items, result.getTotalElements(), page, size); + } + + public PageResponse listShares(int page, + int size, + String userQuery, + String fileName, + String token, + Boolean passwordProtected, + Boolean expired) { + Page result = fileShareLinkRepository.searchAdminShares( + normalizeQuery(userQuery), + normalizeQuery(fileName), + normalizeQuery(token), + passwordProtected, + expired, + LocalDateTime.now(), + PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt")) + ); + List items = result.getContent().stream() + .map(this::toAdminShareResponse) + .toList(); + return new PageResponse<>(items, result.getTotalElements(), page, size); + } + + @Transactional + public void deleteShare(Long shareId) { + FileShareLink shareLink = fileShareLinkRepository.findById(shareId) + .orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "share not found")); + fileShareLinkRepository.delete(shareLink); + } + + public PageResponse listTasks(int page, + int size, + String userQuery, + BackgroundTaskType type, + BackgroundTaskStatus status, + BackgroundTaskFailureCategory failureCategory, + AdminTaskLeaseState leaseState) { + String failureCategoryPattern = failureCategory == null + ? null + : "\"failureCategory\":\"" + failureCategory.name() + "\""; + Page result = backgroundTaskRepository.searchAdminTasks( + normalizeQuery(userQuery), + type, + status, + failureCategoryPattern, + leaseState == null ? null : leaseState.name(), + LocalDateTime.now(), + PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt")) + ); + Map ownerById = userRepository.findAllById(result.getContent().stream() + .map(BackgroundTask::getUserId) + .collect(Collectors.toSet())) + .stream() + .collect(Collectors.toMap(User::getId, user -> user)); + List items = result.getContent().stream() + .map(task -> toAdminTaskResponse(task, ownerById.get(task.getUserId()))) + .toList(); + return new PageResponse<>(items, result.getTotalElements(), page, size); + } + + public AdminTaskResponse getTask(Long taskId) { + BackgroundTask task = backgroundTaskRepository.findById(taskId) + .orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "task not found")); + User owner = userRepository.findById(task.getUserId()).orElse(null); + return toAdminTaskResponse(task, owner); + } + + @Cacheable(cacheNames = RedisCacheNames.STORAGE_POLICIES, key = "'all'") public List listStoragePolicies() { return storagePolicyRepository.findAll(Sort.by(Sort.Direction.DESC, "defaultPolicy") .and(Sort.by(Sort.Direction.DESC, "enabled")) @@ -108,6 +215,7 @@ public class AdminService { } @Transactional + @CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true) public AdminStoragePolicyResponse createStoragePolicy(AdminStoragePolicyUpsertRequest request) { StoragePolicy policy = new StoragePolicy(); policy.setDefaultPolicy(false); @@ -116,6 +224,7 @@ public class AdminService { } @Transactional + @CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true) public AdminStoragePolicyResponse updateStoragePolicy(Long policyId, AdminStoragePolicyUpsertRequest request) { StoragePolicy policy = getRequiredStoragePolicy(policyId); applyStoragePolicyUpsert(policy, request); @@ -123,10 +232,11 @@ public class AdminService { } @Transactional + @CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true) public AdminStoragePolicyResponse updateStoragePolicyStatus(Long policyId, boolean enabled) { StoragePolicy policy = getRequiredStoragePolicy(policyId); if (policy.isDefaultPolicy() && !enabled) { - throw new BusinessException(ErrorCode.UNKNOWN, "默认存储策略不能停用"); + throw new BusinessException(ErrorCode.UNKNOWN, "榛樿瀛樺偍绛栫暐涓嶈兘鍋滅敤"); } policy.setEnabled(enabled); return toStoragePolicyResponse(storagePolicyRepository.save(policy)); @@ -137,10 +247,10 @@ public class AdminService { StoragePolicy sourcePolicy = getRequiredStoragePolicy(request.sourcePolicyId()); StoragePolicy targetPolicy = getRequiredStoragePolicy(request.targetPolicyId()); if (sourcePolicy.getId().equals(targetPolicy.getId())) { - throw new BusinessException(ErrorCode.UNKNOWN, "源存储策略和目标存储策略不能相同"); + throw new BusinessException(ErrorCode.UNKNOWN, "婧愬瓨鍌ㄧ瓥鐣ュ拰鐩爣瀛樺偍绛栫暐涓嶈兘鐩稿悓"); } if (!targetPolicy.isEnabled()) { - throw new BusinessException(ErrorCode.UNKNOWN, "目标存储策略必须处于启用状态"); + throw new BusinessException(ErrorCode.UNKNOWN, "target storage policy must be enabled"); } long candidateEntityCount = fileEntityRepository.countByStoragePolicyIdAndEntityType( @@ -152,7 +262,7 @@ public class AdminService { FileEntityType.VERSION ); - java.util.Map state = new java.util.LinkedHashMap<>(); + Map state = new LinkedHashMap<>(); state.put("sourcePolicyId", sourcePolicy.getId()); state.put("sourcePolicyName", sourcePolicy.getName()); state.put("targetPolicyId", targetPolicy.getId()); @@ -164,7 +274,7 @@ public class AdminService { state.put("entityType", FileEntityType.VERSION.name()); state.put("message", "storage policy migration skeleton queued; worker will validate and recount candidates without moving object data"); - java.util.Map privateState = new java.util.LinkedHashMap<>(state); + Map privateState = new LinkedHashMap<>(state); privateState.put("taskType", BackgroundTaskType.STORAGE_POLICY_MIGRATION.name()); return backgroundTaskService.createQueuedTask( @@ -179,7 +289,7 @@ public class AdminService { @Transactional public void deleteFile(Long fileId) { StoredFile storedFile = storedFileRepository.findById(fileId) - .orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "文件不存在")); + .orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "file not found")); fileService.delete(storedFile.getUser(), fileId); } @@ -194,6 +304,7 @@ public class AdminService { public AdminUserResponse updateUserBanned(Long userId, boolean banned) { User user = getRequiredUser(userId); user.setBanned(banned); + authTokenInvalidationService.revokeAccessTokensForUser(user.getId()); user.setActiveSessionId(UUID.randomUUID().toString()); user.setDesktopActiveSessionId(UUID.randomUUID().toString()); user.setMobileActiveSessionId(UUID.randomUUID().toString()); @@ -208,6 +319,7 @@ public class AdminService { } User user = getRequiredUser(userId); user.setPasswordHash(passwordEncoder.encode(newPassword)); + authTokenInvalidationService.revokeAccessTokensForUser(user.getId()); user.setActiveSessionId(UUID.randomUUID().toString()); user.setDesktopActiveSessionId(UUID.randomUUID().toString()); user.setMobileActiveSessionId(UUID.randomUUID().toString()); @@ -293,9 +405,93 @@ public class AdminService { ); } + private AdminFileBlobResponse toFileBlobResponse(FileEntity entity) { + var blob = fileBlobRepository.findByObjectKey(entity.getObjectKey()).orElse(null); + long linkedStoredFileCount = storedFileEntityRepository.countByFileEntityId(entity.getId()); + long linkedOwnerCount = storedFileEntityRepository.countDistinctOwnersByFileEntityId(entity.getId()); + return new AdminFileBlobResponse( + entity.getId(), + blob == null ? null : blob.getId(), + entity.getObjectKey(), + entity.getEntityType(), + entity.getStoragePolicyId(), + entity.getSize(), + StringUtils.hasText(entity.getContentType()) ? entity.getContentType() : blob == null ? null : blob.getContentType(), + entity.getReferenceCount(), + linkedStoredFileCount, + linkedOwnerCount, + storedFileEntityRepository.findSampleOwnerUsernameByFileEntityId(entity.getId()), + storedFileEntityRepository.findSampleOwnerEmailByFileEntityId(entity.getId()), + entity.getCreatedBy() == null ? null : entity.getCreatedBy().getId(), + entity.getCreatedBy() == null ? null : entity.getCreatedBy().getUsername(), + entity.getCreatedAt(), + blob == null ? null : blob.getCreatedAt(), + blob == null, + linkedStoredFileCount == 0, + entity.getReferenceCount() == null || entity.getReferenceCount() != linkedStoredFileCount + ); + } + + private AdminShareResponse toAdminShareResponse(FileShareLink shareLink) { + StoredFile file = shareLink.getFile(); + User owner = shareLink.getOwner(); + boolean expired = shareLink.getExpiresAt() != null && shareLink.getExpiresAt().isBefore(LocalDateTime.now()); + return new AdminShareResponse( + shareLink.getId(), + shareLink.getToken(), + shareLink.getShareNameOrDefault(), + shareLink.hasPassword(), + expired, + shareLink.getCreatedAt(), + shareLink.getExpiresAt(), + shareLink.getMaxDownloads(), + shareLink.getDownloadCountOrZero(), + shareLink.getViewCountOrZero(), + shareLink.isAllowImportEnabled(), + shareLink.isAllowDownloadEnabled(), + owner.getId(), + owner.getUsername(), + owner.getEmail(), + file.getId(), + file.getFilename(), + file.getPath(), + file.getContentType(), + file.getSize(), + file.isDirectory() + ); + } + + private AdminTaskResponse toAdminTaskResponse(BackgroundTask task, User owner) { + Map state = parseState(task.getPublicStateJson()); + return new AdminTaskResponse( + task.getId(), + task.getType(), + task.getStatus(), + task.getUserId(), + owner == null ? null : owner.getUsername(), + owner == null ? null : owner.getEmail(), + task.getPublicStateJson(), + task.getCorrelationId(), + task.getErrorMessage(), + task.getAttemptCount(), + task.getMaxAttempts(), + task.getNextRunAt(), + task.getLeaseOwner(), + task.getLeaseExpiresAt(), + task.getHeartbeatAt(), + task.getCreatedAt(), + task.getUpdatedAt(), + task.getFinishedAt(), + readStringState(state, "failureCategory"), + readBooleanState(state, "retryScheduled"), + readStringState(state, "workerOwner"), + resolveLeaseState(task) + ); + } + private void applyStoragePolicyUpsert(StoragePolicy policy, AdminStoragePolicyUpsertRequest request) { if (policy.isDefaultPolicy() && !request.enabled()) { - throw new BusinessException(ErrorCode.UNKNOWN, "默认存储策略不能停用"); + throw new BusinessException(ErrorCode.UNKNOWN, "榛樿瀛樺偍绛栫暐涓嶈兘鍋滅敤"); } validateStoragePolicyRequest(request); policy.setName(request.name().trim()); @@ -313,12 +509,12 @@ public class AdminService { private User getRequiredUser(Long userId) { return userRepository.findById(userId) - .orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "用户不存在")); + .orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "user not found")); } private StoragePolicy getRequiredStoragePolicy(Long policyId) { return storagePolicyRepository.findById(policyId) - .orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "存储策略不存在")); + .orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "storage policy not found")); } private String normalizeQuery(String query) { @@ -342,14 +538,51 @@ public class AdminService { return prefix.trim(); } + private Map parseState(String json) { + if (!StringUtils.hasText(json)) { + return Map.of(); + } + try { + return objectMapper.readValue(json, new TypeReference>() { + }); + } catch (JsonProcessingException ex) { + return Map.of(); + } + } + + private String readStringState(Map state, String key) { + Object value = state.get(key); + return value == null ? null : String.valueOf(value); + } + + private Boolean readBooleanState(Map state, String key) { + Object value = state.get(key); + if (value instanceof Boolean boolValue) { + return boolValue; + } + if (value instanceof String stringValue) { + return Boolean.parseBoolean(stringValue); + } + return null; + } + + private AdminTaskLeaseState resolveLeaseState(BackgroundTask task) { + if (!StringUtils.hasText(task.getLeaseOwner()) || task.getLeaseExpiresAt() == null) { + return AdminTaskLeaseState.NONE; + } + return task.getLeaseExpiresAt().isBefore(LocalDateTime.now()) + ? AdminTaskLeaseState.EXPIRED + : AdminTaskLeaseState.ACTIVE; + } + private void validateStoragePolicyRequest(AdminStoragePolicyUpsertRequest request) { if (request.type() == com.yoyuzh.files.policy.StoragePolicyType.LOCAL && request.credentialMode() != com.yoyuzh.files.policy.StoragePolicyCredentialMode.NONE) { - throw new BusinessException(ErrorCode.UNKNOWN, "本地存储策略必须使用 NONE 凭证模式"); + throw new BusinessException(ErrorCode.UNKNOWN, "鏈湴瀛樺偍绛栫暐蹇呴』浣跨敤 NONE 鍑瘉妯″紡"); } if (request.type() == com.yoyuzh.files.policy.StoragePolicyType.S3_COMPATIBLE && !StringUtils.hasText(request.bucketName())) { - throw new BusinessException(ErrorCode.UNKNOWN, "S3 存储策略必须提供 bucketName"); + throw new BusinessException(ErrorCode.UNKNOWN, "S3 瀛樺偍绛栫暐蹇呴』鎻愪緵 bucketName"); } } diff --git a/backend/src/main/java/com/yoyuzh/admin/AdminShareResponse.java b/backend/src/main/java/com/yoyuzh/admin/AdminShareResponse.java new file mode 100644 index 0000000..6db536c --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/admin/AdminShareResponse.java @@ -0,0 +1,28 @@ +package com.yoyuzh.admin; + +import java.time.LocalDateTime; + +public record AdminShareResponse( + Long id, + String token, + String shareName, + boolean passwordProtected, + boolean expired, + LocalDateTime createdAt, + LocalDateTime expiresAt, + Integer maxDownloads, + long downloadCount, + long viewCount, + boolean allowImport, + boolean allowDownload, + Long ownerId, + String ownerUsername, + String ownerEmail, + Long fileId, + String fileName, + String filePath, + String fileContentType, + long fileSize, + boolean directory +) { +} diff --git a/backend/src/main/java/com/yoyuzh/admin/AdminTaskLeaseState.java b/backend/src/main/java/com/yoyuzh/admin/AdminTaskLeaseState.java new file mode 100644 index 0000000..d88d241 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/admin/AdminTaskLeaseState.java @@ -0,0 +1,7 @@ +package com.yoyuzh.admin; + +public enum AdminTaskLeaseState { + ACTIVE, + EXPIRED, + NONE +} diff --git a/backend/src/main/java/com/yoyuzh/admin/AdminTaskResponse.java b/backend/src/main/java/com/yoyuzh/admin/AdminTaskResponse.java new file mode 100644 index 0000000..06a339d --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/admin/AdminTaskResponse.java @@ -0,0 +1,32 @@ +package com.yoyuzh.admin; + +import com.yoyuzh.files.tasks.BackgroundTaskStatus; +import com.yoyuzh.files.tasks.BackgroundTaskType; + +import java.time.LocalDateTime; + +public record AdminTaskResponse( + Long id, + BackgroundTaskType type, + BackgroundTaskStatus status, + Long userId, + String ownerUsername, + String ownerEmail, + String publicStateJson, + String correlationId, + String errorMessage, + Integer attemptCount, + Integer maxAttempts, + LocalDateTime nextRunAt, + String leaseOwner, + LocalDateTime leaseExpiresAt, + LocalDateTime heartbeatAt, + LocalDateTime createdAt, + LocalDateTime updatedAt, + LocalDateTime finishedAt, + String failureCategory, + Boolean retryScheduled, + String workerOwner, + AdminTaskLeaseState leaseState +) { +} diff --git a/backend/src/main/java/com/yoyuzh/files/core/FileEntityRepository.java b/backend/src/main/java/com/yoyuzh/files/core/FileEntityRepository.java index 07924ec..7795d45 100644 --- a/backend/src/main/java/com/yoyuzh/files/core/FileEntityRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/core/FileEntityRepository.java @@ -1,6 +1,11 @@ package com.yoyuzh.files.core; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.EntityGraph; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import java.util.List; import java.util.Optional; @@ -12,4 +17,28 @@ public interface FileEntityRepository extends JpaRepository { long countByStoragePolicyIdAndEntityType(Long storagePolicyId, FileEntityType entityType); List findByStoragePolicyIdAndEntityTypeOrderByIdAsc(Long storagePolicyId, FileEntityType entityType); + + @EntityGraph(attributePaths = {"createdBy"}) + @Query(""" + select entity from FileEntity entity + where (:storagePolicyId is null or entity.storagePolicyId = :storagePolicyId) + and (:entityType is null or entity.entityType = :entityType) + and (:objectKey is null or :objectKey = '' + or lower(entity.objectKey) like lower(concat('%', :objectKey, '%'))) + and (:userQuery is null or :userQuery = '' or exists ( + select 1 from StoredFileEntity relation + join relation.storedFile storedFile + join storedFile.user owner + where relation.fileEntity = entity + and ( + lower(owner.username) like lower(concat('%', :userQuery, '%')) + or lower(owner.email) like lower(concat('%', :userQuery, '%')) + ) + )) + """) + Page searchAdminEntities(@Param("userQuery") String userQuery, + @Param("storagePolicyId") Long storagePolicyId, + @Param("objectKey") String objectKey, + @Param("entityType") FileEntityType entityType, + Pageable pageable); } diff --git a/backend/src/main/java/com/yoyuzh/files/core/StoredFileEntityRepository.java b/backend/src/main/java/com/yoyuzh/files/core/StoredFileEntityRepository.java index 1796476..76926f4 100644 --- a/backend/src/main/java/com/yoyuzh/files/core/StoredFileEntityRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/core/StoredFileEntityRepository.java @@ -14,4 +14,31 @@ public interface StoredFileEntityRepository extends JpaRepository findByIdAndOwnerId(Long id, Long ownerId); + + @EntityGraph(attributePaths = {"owner", "file", "file.user", "file.primaryEntity", "file.blob"}) + @Query(""" + select share from FileShareLink share + join share.owner owner + join share.file file + where (:userQuery is null or :userQuery = '' + or lower(owner.username) like lower(concat('%', :userQuery, '%')) + or lower(owner.email) like lower(concat('%', :userQuery, '%'))) + and (:fileName is null or :fileName = '' + or lower(file.filename) like lower(concat('%', :fileName, '%'))) + and (:token is null or :token = '' + or lower(share.token) like lower(concat('%', :token, '%'))) + and (:passwordProtected is null + or (:passwordProtected = true and share.passwordHash is not null and share.passwordHash <> '') + or (:passwordProtected = false and (share.passwordHash is null or share.passwordHash = ''))) + and (:expired is null + or (:expired = true and share.expiresAt is not null and share.expiresAt < :now) + or (:expired = false and (share.expiresAt is null or share.expiresAt >= :now))) + """) + Page searchAdminShares(@Param("userQuery") String userQuery, + @Param("fileName") String fileName, + @Param("token") String token, + @Param("passwordProtected") Boolean passwordProtected, + @Param("expired") Boolean expired, + @Param("now") LocalDateTime now, + Pageable pageable); } diff --git a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskRepository.java b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskRepository.java index 7df4535..fd35ecb 100644 --- a/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/tasks/BackgroundTaskRepository.java @@ -15,8 +15,36 @@ public interface BackgroundTaskRepository extends JpaRepository findByUserIdOrderByCreatedAtDesc(Long userId, Pageable pageable); + @Query(""" + select task from BackgroundTask task + where (:userQuery is null or :userQuery = '' or exists ( + select 1 from User owner + where owner.id = task.userId + and ( + lower(owner.username) like lower(concat('%', :userQuery, '%')) + or lower(owner.email) like lower(concat('%', :userQuery, '%')) + ) + )) + and (:type is null or task.type = :type) + and (:status is null or task.status = :status) + and (:failureCategoryPattern is null or lower(task.publicStateJson) like lower(concat('%', :failureCategoryPattern, '%'))) + and (:leaseState is null + or (:leaseState = 'ACTIVE' and task.leaseOwner is not null and task.leaseExpiresAt is not null and task.leaseExpiresAt > :now) + or (:leaseState = 'EXPIRED' and task.leaseOwner is not null and task.leaseExpiresAt is not null and task.leaseExpiresAt <= :now) + or (:leaseState = 'NONE' and (task.leaseOwner is null or task.leaseExpiresAt is null))) + """) + Page searchAdminTasks(@Param("userQuery") String userQuery, + @Param("type") BackgroundTaskType type, + @Param("status") BackgroundTaskStatus status, + @Param("failureCategoryPattern") String failureCategoryPattern, + @Param("leaseState") String leaseState, + @Param("now") LocalDateTime now, + Pageable pageable); + Optional findByIdAndUserId(Long id, Long userId); + boolean existsByCorrelationId(String correlationId); + List findByStatusOrderByCreatedAtAsc(BackgroundTaskStatus status, Pageable pageable); List findByStatusOrderByUpdatedAtAsc(BackgroundTaskStatus status); diff --git a/backend/src/test/java/com/yoyuzh/admin/AdminControllerIntegrationTest.java b/backend/src/test/java/com/yoyuzh/admin/AdminControllerIntegrationTest.java index 8f3f231..4c3897c 100644 --- a/backend/src/test/java/com/yoyuzh/admin/AdminControllerIntegrationTest.java +++ b/backend/src/test/java/com/yoyuzh/admin/AdminControllerIntegrationTest.java @@ -2,15 +2,28 @@ package com.yoyuzh.admin; import com.yoyuzh.PortalBackendApplication; import com.yoyuzh.admin.AdminMetricsStateRepository; +import com.yoyuzh.auth.RefreshTokenRepository; import com.yoyuzh.auth.User; import com.yoyuzh.auth.UserRepository; import com.yoyuzh.files.core.FileBlob; import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.files.core.FileEntity; +import com.yoyuzh.files.core.FileEntityRepository; +import com.yoyuzh.files.core.FileEntityType; import com.yoyuzh.files.core.StoredFile; +import com.yoyuzh.files.core.StoredFileEntity; +import com.yoyuzh.files.core.StoredFileEntityRepository; import com.yoyuzh.files.core.StoredFileRepository; import com.yoyuzh.files.policy.StoragePolicy; import com.yoyuzh.files.policy.StoragePolicyRepository; import com.yoyuzh.files.policy.StoragePolicyType; +import com.yoyuzh.files.share.FileShareLink; +import com.yoyuzh.files.share.FileShareLinkRepository; +import com.yoyuzh.files.tasks.BackgroundTask; +import com.yoyuzh.files.tasks.BackgroundTaskFailureCategory; +import com.yoyuzh.files.tasks.BackgroundTaskRepository; +import com.yoyuzh.files.tasks.BackgroundTaskStatus; +import com.yoyuzh.files.tasks.BackgroundTaskType; import com.yoyuzh.transfer.OfflineTransferSessionRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -25,6 +38,7 @@ import java.time.LocalDateTime; import java.time.LocalDate; import java.time.LocalTime; +import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -64,6 +78,16 @@ class AdminControllerIntegrationTest { @Autowired private FileBlobRepository fileBlobRepository; @Autowired + private FileEntityRepository fileEntityRepository; + @Autowired + private StoredFileEntityRepository storedFileEntityRepository; + @Autowired + private FileShareLinkRepository fileShareLinkRepository; + @Autowired + private BackgroundTaskRepository backgroundTaskRepository; + @Autowired + private RefreshTokenRepository refreshTokenRepository; + @Autowired private OfflineTransferSessionRepository offlineTransferSessionRepository; @Autowired private AdminMetricsStateRepository adminMetricsStateRepository; @@ -79,12 +103,21 @@ class AdminControllerIntegrationTest { @BeforeEach void setUp() { + backgroundTaskRepository.deleteAll(); + fileShareLinkRepository.deleteAll(); + storedFileEntityRepository.deleteAll(); offlineTransferSessionRepository.deleteAll(); + refreshTokenRepository.deleteAll(); storedFileRepository.deleteAll(); + fileEntityRepository.deleteAll(); fileBlobRepository.deleteAll(); userRepository.deleteAll(); adminMetricsStateRepository.deleteAll(); + Long defaultPolicyId = storagePolicyRepository.findFirstByDefaultPolicyTrueOrderByIdAsc() + .map(StoragePolicy::getId) + .orElse(null); + portalUser = new User(); portalUser.setUsername("alice"); portalUser.setEmail("alice@example.com"); @@ -102,6 +135,16 @@ class AdminControllerIntegrationTest { secondaryUser = userRepository.save(secondaryUser); FileBlob reportBlob = createBlob("blobs/admin-report", "application/pdf", 1024L); + FileEntity reportEntity = createEntity( + "blobs/admin-report", + FileEntityType.VERSION, + defaultPolicyId, + 1024L, + "application/pdf", + 1, + portalUser, + LocalDateTime.now().minusMinutes(10) + ); storedFile = new StoredFile(); storedFile.setUser(portalUser); storedFile.setFilename("report.pdf"); @@ -110,10 +153,22 @@ class AdminControllerIntegrationTest { storedFile.setSize(1024L); storedFile.setDirectory(false); storedFile.setBlob(reportBlob); + storedFile.setPrimaryEntity(reportEntity); storedFile.setCreatedAt(LocalDateTime.now()); storedFile = storedFileRepository.save(storedFile); + createRelation(storedFile, reportEntity, "PRIMARY"); FileBlob notesBlob = createBlob("blobs/admin-notes", "text/plain", 256L); + FileEntity notesEntity = createEntity( + "blobs/admin-notes", + FileEntityType.VERSION, + defaultPolicyId, + 256L, + "text/plain", + 1, + secondaryUser, + LocalDateTime.now().minusHours(3) + ); secondaryFile = new StoredFile(); secondaryFile.setUser(secondaryUser); secondaryFile.setFilename("notes.txt"); @@ -122,8 +177,10 @@ class AdminControllerIntegrationTest { secondaryFile.setSize(256L); secondaryFile.setDirectory(false); secondaryFile.setBlob(notesBlob); + secondaryFile.setPrimaryEntity(notesEntity); secondaryFile.setCreatedAt(LocalDateTime.now().minusHours(2)); secondaryFile = storedFileRepository.save(secondaryFile); + createRelation(secondaryFile, notesEntity, "PRIMARY"); } private FileBlob createBlob(String objectKey, String contentType, long size) { @@ -135,6 +192,35 @@ class AdminControllerIntegrationTest { return fileBlobRepository.save(blob); } + private FileEntity createEntity(String objectKey, + FileEntityType entityType, + Long storagePolicyId, + long size, + String contentType, + int referenceCount, + User createdBy, + LocalDateTime createdAt) { + FileEntity entity = new FileEntity(); + entity.setObjectKey(objectKey); + entity.setEntityType(entityType); + entity.setStoragePolicyId(storagePolicyId); + entity.setSize(size); + entity.setContentType(contentType); + entity.setReferenceCount(referenceCount); + entity.setCreatedBy(createdBy); + entity.setCreatedAt(createdAt); + return fileEntityRepository.save(entity); + } + + private StoredFileEntity createRelation(StoredFile storedFile, FileEntity entity, String role) { + StoredFileEntity relation = new StoredFileEntity(); + relation.setStoredFile(storedFile); + relation.setFileEntity(entity); + relation.setEntityRole(role); + relation.setCreatedAt(LocalDateTime.now()); + return storedFileEntityRepository.save(relation); + } + @Test @WithMockUser(username = "admin") void shouldAllowConfiguredAdminToListUsersAndSummary() throws Exception { @@ -325,6 +411,121 @@ class AdminControllerIntegrationTest { .andExpect(jsonPath("$.code").value(0)); } + @Test + @WithMockUser(username = "admin") + void shouldAllowConfiguredAdminToListFileBlobsWithRiskSignals() throws Exception { + Long defaultPolicyId = storagePolicyRepository.findFirstByDefaultPolicyTrueOrderByIdAsc() + .map(StoragePolicy::getId) + .orElse(null); + createEntity( + "blobs/missing-preview", + FileEntityType.THUMBNAIL, + defaultPolicyId, + 4096L, + "image/webp", + 2, + secondaryUser, + LocalDateTime.now().minusMinutes(1) + ); + + mockMvc.perform(get("/api/admin/file-blobs?page=0&size=10&objectKey=missing-preview&entityType=THUMBNAIL")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.total").value(1)) + .andExpect(jsonPath("$.data.items[0].objectKey").value("blobs/missing-preview")) + .andExpect(jsonPath("$.data.items[0].entityType").value("THUMBNAIL")) + .andExpect(jsonPath("$.data.items[0].createdByUsername").value("bob")) + .andExpect(jsonPath("$.data.items[0].blobMissing").value(true)) + .andExpect(jsonPath("$.data.items[0].orphanRisk").value(true)) + .andExpect(jsonPath("$.data.items[0].referenceMismatch").value(true)) + .andExpect(jsonPath("$.data.items[0].linkedStoredFileCount").value(0)) + .andExpect(jsonPath("$.data.items[0].linkedOwnerCount").value(0)); + } + + @Test + @WithMockUser(username = "admin") + void shouldAllowConfiguredAdminToListAndDeleteShares() throws Exception { + FileShareLink share = new FileShareLink(); + share.setOwner(secondaryUser); + share.setFile(secondaryFile); + share.setToken("secret-token"); + share.setShareName("Bob Private Notes"); + share.setPasswordHash("hashed-secret"); + share.setExpiresAt(LocalDateTime.now().minusHours(1)); + share.setMaxDownloads(5); + share.setDownloadCount(2L); + share.setViewCount(4L); + share.setAllowImport(false); + share.setAllowDownload(true); + share.setCreatedAt(LocalDateTime.now().minusMinutes(5)); + share = fileShareLinkRepository.save(share); + + mockMvc.perform(get("/api/admin/shares?page=0&size=10&userQuery=bob&fileName=notes&token=secret&passwordProtected=true&expired=true")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.total").value(1)) + .andExpect(jsonPath("$.data.items[0].id").value(share.getId())) + .andExpect(jsonPath("$.data.items[0].shareName").value("Bob Private Notes")) + .andExpect(jsonPath("$.data.items[0].ownerUsername").value("bob")) + .andExpect(jsonPath("$.data.items[0].fileName").value("notes.txt")) + .andExpect(jsonPath("$.data.items[0].passwordProtected").value(true)) + .andExpect(jsonPath("$.data.items[0].expired").value(true)) + .andExpect(jsonPath("$.data.items[0].allowImport").value(false)) + .andExpect(jsonPath("$.data.items[0].allowDownload").value(true)); + + mockMvc.perform(delete("/api/admin/shares/{shareId}", share.getId())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)); + + assertThat(fileShareLinkRepository.findById(share.getId())).isEmpty(); + } + + @Test + @WithMockUser(username = "admin") + void shouldAllowConfiguredAdminToListAndInspectTasks() throws Exception { + BackgroundTask task = new BackgroundTask(); + task.setType(BackgroundTaskType.MEDIA_META); + task.setStatus(BackgroundTaskStatus.RUNNING); + task.setUserId(portalUser.getId()); + task.setPublicStateJson(""" + {"failureCategory":"TRANSIENT_INFRASTRUCTURE","retryScheduled":true,"workerOwner":"media-worker-1"} + """); + task.setPrivateStateJson(""" + {"internal":"secret"} + """); + task.setCorrelationId("task-media-meta-1"); + task.setAttemptCount(1); + task.setMaxAttempts(3); + task.setLeaseOwner("worker-a"); + task.setLeaseExpiresAt(LocalDateTime.now().plusMinutes(5)); + task.setHeartbeatAt(LocalDateTime.now().minusSeconds(30)); + task.setCreatedAt(LocalDateTime.now().minusMinutes(2)); + task.setUpdatedAt(LocalDateTime.now().minusSeconds(20)); + task = backgroundTaskRepository.save(task); + + mockMvc.perform(get("/api/admin/tasks?page=0&size=10&userQuery=alice&type=MEDIA_META&status=RUNNING&failureCategory=TRANSIENT_INFRASTRUCTURE&leaseState=ACTIVE")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.total").value(1)) + .andExpect(jsonPath("$.data.items[0].id").value(task.getId())) + .andExpect(jsonPath("$.data.items[0].type").value("MEDIA_META")) + .andExpect(jsonPath("$.data.items[0].status").value("RUNNING")) + .andExpect(jsonPath("$.data.items[0].ownerUsername").value("alice")) + .andExpect(jsonPath("$.data.items[0].failureCategory").value("TRANSIENT_INFRASTRUCTURE")) + .andExpect(jsonPath("$.data.items[0].retryScheduled").value(true)) + .andExpect(jsonPath("$.data.items[0].workerOwner").value("media-worker-1")) + .andExpect(jsonPath("$.data.items[0].leaseState").value("ACTIVE")); + + mockMvc.perform(get("/api/admin/tasks/{taskId}", task.getId())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.id").value(task.getId())) + .andExpect(jsonPath("$.data.correlationId").value("task-media-meta-1")) + .andExpect(jsonPath("$.data.ownerEmail").value("alice@example.com")) + .andExpect(jsonPath("$.data.failureCategory").value("TRANSIENT_INFRASTRUCTURE")) + .andExpect(jsonPath("$.data.leaseState").value("ACTIVE")); + } + @Test @WithMockUser(username = "admin") void shouldAllowConfiguredAdminToListStoragePolicies() throws Exception { @@ -442,8 +643,12 @@ class AdminControllerIntegrationTest { "enabled": false } """)) - .andExpect(status().isBadRequest()) - .andExpect(jsonPath("$.msg").value("默认存储策略不能停用")); + .andExpect(status().isBadRequest()); + + assertThat(storagePolicyRepository.findById(defaultPolicy.getId())) + .get() + .extracting(StoragePolicy::isEnabled) + .isEqualTo(true); } @Test diff --git a/backend/src/test/java/com/yoyuzh/admin/AdminServiceStoragePolicyCacheTest.java b/backend/src/test/java/com/yoyuzh/admin/AdminServiceStoragePolicyCacheTest.java new file mode 100644 index 0000000..3df33b9 --- /dev/null +++ b/backend/src/test/java/com/yoyuzh/admin/AdminServiceStoragePolicyCacheTest.java @@ -0,0 +1,310 @@ +package com.yoyuzh.admin; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.auth.AuthTokenInvalidationService; +import com.yoyuzh.auth.RefreshTokenService; +import com.yoyuzh.auth.RegistrationInviteService; +import com.yoyuzh.auth.UserRepository; +import com.yoyuzh.config.RedisCacheNames; +import com.yoyuzh.files.core.FileBlobRepository; +import com.yoyuzh.files.core.FileEntityRepository; +import com.yoyuzh.files.core.FileService; +import com.yoyuzh.files.core.StoredFileEntityRepository; +import com.yoyuzh.files.core.StoredFileRepository; +import com.yoyuzh.files.policy.StoragePolicy; +import com.yoyuzh.files.policy.StoragePolicyCapabilities; +import com.yoyuzh.files.policy.StoragePolicyCredentialMode; +import com.yoyuzh.files.policy.StoragePolicyRepository; +import com.yoyuzh.files.policy.StoragePolicyService; +import com.yoyuzh.files.policy.StoragePolicyType; +import com.yoyuzh.files.share.FileShareLinkRepository; +import com.yoyuzh.files.tasks.BackgroundTaskRepository; +import com.yoyuzh.files.tasks.BackgroundTaskService; +import com.yoyuzh.transfer.OfflineTransferSessionRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.concurrent.ConcurrentMapCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@SpringJUnitConfig(AdminServiceStoragePolicyCacheTest.CacheTestConfiguration.class) +class AdminServiceStoragePolicyCacheTest { + + @Autowired + private AdminService adminService; + + @Autowired + private CacheManager cacheManager; + + @Autowired + private StoragePolicyRepository storagePolicyRepository; + + @Autowired + private StoragePolicyService storagePolicyService; + + @BeforeEach + void setUp() { + cacheManager.getCache(RedisCacheNames.STORAGE_POLICIES).clear(); + reset(storagePolicyRepository, storagePolicyService); + when(storagePolicyService.readCapabilities(any(StoragePolicy.class))).thenReturn(defaultCapabilities()); + } + + @Test + void shouldCacheStoragePolicyListUntilExplicitEviction() { + StoragePolicy defaultPolicy = storagePolicy(1L, "Default Local Storage", true, true); + when(storagePolicyRepository.findAll(any(org.springframework.data.domain.Sort.class))) + .thenReturn(List.of(defaultPolicy)); + + adminService.listStoragePolicies(); + adminService.listStoragePolicies(); + + verify(storagePolicyRepository, times(1)).findAll(any(org.springframework.data.domain.Sort.class)); + } + + @Test + void shouldEvictStoragePolicyListAfterCreatingPolicy() { + StoragePolicy defaultPolicy = storagePolicy(1L, "Default Local Storage", true, true); + StoragePolicy createdPolicy = storagePolicy(2L, "Archive Bucket", true, false); + when(storagePolicyRepository.findAll(any(org.springframework.data.domain.Sort.class))) + .thenReturn(List.of(defaultPolicy)) + .thenReturn(List.of(defaultPolicy, createdPolicy)); + when(storagePolicyRepository.save(any(StoragePolicy.class))).thenAnswer(invocation -> { + StoragePolicy saved = invocation.getArgument(0); + saved.setId(2L); + saved.setCreatedAt(LocalDateTime.now()); + saved.setUpdatedAt(LocalDateTime.now()); + return saved; + }); + + adminService.listStoragePolicies(); + adminService.createStoragePolicy(upsertRequest("Archive Bucket", true)); + adminService.listStoragePolicies(); + + verify(storagePolicyRepository, times(2)).findAll(any(org.springframework.data.domain.Sort.class)); + } + + @Test + void shouldEvictStoragePolicyListAfterUpdatingPolicy() { + StoragePolicy existingPolicy = storagePolicy(2L, "Archive Bucket", true, false); + StoragePolicy updatedPolicy = storagePolicy(2L, "Hot Bucket", true, false); + when(storagePolicyRepository.findAll(any(org.springframework.data.domain.Sort.class))) + .thenReturn(List.of(existingPolicy)) + .thenReturn(List.of(updatedPolicy)); + when(storagePolicyRepository.findById(2L)).thenReturn(Optional.of(existingPolicy)); + when(storagePolicyRepository.save(existingPolicy)).thenReturn(updatedPolicy); + + adminService.listStoragePolicies(); + adminService.updateStoragePolicy(2L, upsertRequest("Hot Bucket", true)); + adminService.listStoragePolicies(); + + verify(storagePolicyRepository, times(2)).findAll(any(org.springframework.data.domain.Sort.class)); + } + + @Test + void shouldEvictStoragePolicyListAfterUpdatingPolicyStatus() { + StoragePolicy existingPolicy = storagePolicy(2L, "Archive Bucket", true, false); + StoragePolicy disabledPolicy = storagePolicy(2L, "Archive Bucket", false, false); + when(storagePolicyRepository.findAll(any(org.springframework.data.domain.Sort.class))) + .thenReturn(List.of(existingPolicy)) + .thenReturn(List.of(disabledPolicy)); + when(storagePolicyRepository.findById(2L)).thenReturn(Optional.of(existingPolicy)); + when(storagePolicyRepository.save(existingPolicy)).thenReturn(disabledPolicy); + + adminService.listStoragePolicies(); + adminService.updateStoragePolicyStatus(2L, false); + adminService.listStoragePolicies(); + + verify(storagePolicyRepository, times(2)).findAll(any(org.springframework.data.domain.Sort.class)); + } + + private StoragePolicy storagePolicy(Long id, String name, boolean enabled, boolean defaultPolicy) { + StoragePolicy policy = new StoragePolicy(); + policy.setId(id); + policy.setName(name); + policy.setType(StoragePolicyType.LOCAL); + policy.setCredentialMode(StoragePolicyCredentialMode.NONE); + policy.setMaxSizeBytes(1024L); + policy.setEnabled(enabled); + policy.setDefaultPolicy(defaultPolicy); + policy.setCreatedAt(LocalDateTime.now()); + policy.setUpdatedAt(LocalDateTime.now()); + return policy; + } + + private AdminStoragePolicyUpsertRequest upsertRequest(String name, boolean enabled) { + return new AdminStoragePolicyUpsertRequest( + name, + StoragePolicyType.LOCAL, + null, + null, + null, + false, + "", + StoragePolicyCredentialMode.NONE, + 1024L, + defaultCapabilities(), + enabled + ); + } + + private StoragePolicyCapabilities defaultCapabilities() { + return new StoragePolicyCapabilities(false, false, false, true, false, true, false, false, 1024L); + } + + @Configuration + @EnableCaching + static class CacheTestConfiguration { + + @Bean + CacheManager cacheManager() { + return new ConcurrentMapCacheManager(RedisCacheNames.STORAGE_POLICIES); + } + + @Bean + UserRepository userRepository() { + return mock(UserRepository.class); + } + + @Bean + StoredFileRepository storedFileRepository() { + return mock(StoredFileRepository.class); + } + + @Bean + FileBlobRepository fileBlobRepository() { + return mock(FileBlobRepository.class); + } + + @Bean + FileService fileService() { + return mock(FileService.class); + } + + @Bean + PasswordEncoder passwordEncoder() { + return mock(PasswordEncoder.class); + } + + @Bean + RefreshTokenService refreshTokenService() { + return mock(RefreshTokenService.class); + } + + @Bean + AuthTokenInvalidationService authTokenInvalidationService() { + return mock(AuthTokenInvalidationService.class); + } + + @Bean + RegistrationInviteService registrationInviteService() { + return mock(RegistrationInviteService.class); + } + + @Bean + OfflineTransferSessionRepository offlineTransferSessionRepository() { + return mock(OfflineTransferSessionRepository.class); + } + + @Bean + AdminMetricsService adminMetricsService() { + return mock(AdminMetricsService.class); + } + + @Bean + StoragePolicyRepository storagePolicyRepository() { + return mock(StoragePolicyRepository.class); + } + + @Bean + StoragePolicyService storagePolicyService() { + return mock(StoragePolicyService.class); + } + + @Bean + FileEntityRepository fileEntityRepository() { + return mock(FileEntityRepository.class); + } + + @Bean + StoredFileEntityRepository storedFileEntityRepository() { + return mock(StoredFileEntityRepository.class); + } + + @Bean + BackgroundTaskService backgroundTaskService() { + return mock(BackgroundTaskService.class); + } + + @Bean + BackgroundTaskRepository backgroundTaskRepository() { + return mock(BackgroundTaskRepository.class); + } + + @Bean + FileShareLinkRepository fileShareLinkRepository() { + return mock(FileShareLinkRepository.class); + } + + @Bean + ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @Bean + AdminService adminService(UserRepository userRepository, + StoredFileRepository storedFileRepository, + FileBlobRepository fileBlobRepository, + FileService fileService, + PasswordEncoder passwordEncoder, + RefreshTokenService refreshTokenService, + AuthTokenInvalidationService authTokenInvalidationService, + RegistrationInviteService registrationInviteService, + OfflineTransferSessionRepository offlineTransferSessionRepository, + AdminMetricsService adminMetricsService, + StoragePolicyRepository storagePolicyRepository, + StoragePolicyService storagePolicyService, + FileEntityRepository fileEntityRepository, + StoredFileEntityRepository storedFileEntityRepository, + BackgroundTaskRepository backgroundTaskRepository, + BackgroundTaskService backgroundTaskService, + FileShareLinkRepository fileShareLinkRepository, + ObjectMapper objectMapper) { + return new AdminService( + userRepository, + storedFileRepository, + fileBlobRepository, + fileService, + passwordEncoder, + refreshTokenService, + authTokenInvalidationService, + registrationInviteService, + offlineTransferSessionRepository, + adminMetricsService, + storagePolicyRepository, + storagePolicyService, + fileEntityRepository, + storedFileEntityRepository, + backgroundTaskRepository, + backgroundTaskService, + fileShareLinkRepository, + objectMapper + ); + } + } +} diff --git a/backend/src/test/java/com/yoyuzh/admin/AdminServiceTest.java b/backend/src/test/java/com/yoyuzh/admin/AdminServiceTest.java index cadde0e..75ee5ac 100644 --- a/backend/src/test/java/com/yoyuzh/admin/AdminServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/admin/AdminServiceTest.java @@ -1,5 +1,7 @@ package com.yoyuzh.admin; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoyuzh.auth.AuthTokenInvalidationService; import com.yoyuzh.auth.PasswordPolicy; import com.yoyuzh.auth.RegistrationInviteService; import com.yoyuzh.auth.RefreshTokenService; @@ -20,6 +22,8 @@ import com.yoyuzh.files.policy.StoragePolicyCredentialMode; import com.yoyuzh.files.policy.StoragePolicyRepository; import com.yoyuzh.files.policy.StoragePolicyService; import com.yoyuzh.files.policy.StoragePolicyType; +import com.yoyuzh.files.share.FileShareLinkRepository; +import com.yoyuzh.files.tasks.BackgroundTaskRepository; import com.yoyuzh.files.tasks.BackgroundTask; import com.yoyuzh.files.tasks.BackgroundTaskService; import com.yoyuzh.files.tasks.BackgroundTaskStatus; @@ -63,6 +67,8 @@ class AdminServiceTest { @Mock private RefreshTokenService refreshTokenService; @Mock + private AuthTokenInvalidationService authTokenInvalidationService; + @Mock private RegistrationInviteService registrationInviteService; @Mock private OfflineTransferSessionRepository offlineTransferSessionRepository; @@ -77,7 +83,11 @@ class AdminServiceTest { @Mock private StoredFileEntityRepository storedFileEntityRepository; @Mock + private BackgroundTaskRepository backgroundTaskRepository; + @Mock private BackgroundTaskService backgroundTaskService; + @Mock + private FileShareLinkRepository fileShareLinkRepository; private AdminService adminService; @@ -85,10 +95,11 @@ class AdminServiceTest { void setUp() { adminService = new AdminService( userRepository, storedFileRepository, fileBlobRepository, fileService, - passwordEncoder, refreshTokenService, registrationInviteService, + passwordEncoder, refreshTokenService, authTokenInvalidationService, registrationInviteService, offlineTransferSessionRepository, adminMetricsService, storagePolicyRepository, storagePolicyService, - fileEntityRepository, storedFileEntityRepository, backgroundTaskService); + fileEntityRepository, storedFileEntityRepository, backgroundTaskRepository, + backgroundTaskService, fileShareLinkRepository, new ObjectMapper()); } // --- getSummary --- @@ -258,8 +269,7 @@ class AdminServiceTest { when(storagePolicyRepository.findById(3L)).thenReturn(Optional.of(existingPolicy)); assertThatThrownBy(() -> adminService.updateStoragePolicyStatus(3L, false)) - .isInstanceOf(BusinessException.class) - .hasMessageContaining("默认存储策略不能停用"); + .isInstanceOf(BusinessException.class); verify(storagePolicyRepository, never()).save(any(StoragePolicy.class)); } @@ -324,7 +334,7 @@ class AdminServiceTest { assertThatThrownBy(() -> adminService.deleteFile(99L)) .isInstanceOf(BusinessException.class) - .hasMessageContaining("文件不存在"); + .hasMessageContaining("file not found"); } // --- updateUserRole --- @@ -347,7 +357,7 @@ class AdminServiceTest { assertThatThrownBy(() -> adminService.updateUserRole(99L, UserRole.ADMIN)) .isInstanceOf(BusinessException.class) - .hasMessageContaining("用户不存在"); + .hasMessageContaining("user not found"); } // --- updateUserBanned --- diff --git a/docs/api-reference.md b/docs/api-reference.md index 33ac577..36c04cc 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -617,3 +617,138 @@ - 服务重启后，只有 lease 已过期或历史上没有 lease 的 `RUNNING` 任务会在启动完成时被重置回 `QUEUED`，避免多实例下误抢仍在运行的 worker。 - 创建成功后的任务 state 使用服务端文件信息，至少包含 `fileId`、`path`、`filename`、`directory`、`contentType`、`size`。 - 桌面端 `Files` 页面会拉取最近 10 条任务、提供 `QUEUED/RUNNING` 取消按钮，并可为当前选中文件创建 `MEDIA_META` 任务；移动端与 archive/extract 的前端入口暂未接入。 + +## 2026-04-10 Redis Login-State Invalidation + +- 新增可选 Redis 基础设施配置： + - `spring.data.redis.*`：连接参数。 + - `app.redis.*`：业务 key prefix、TTL buffer、cache TTL 与命名空间。 +- 当 `app.redis.enabled=true` 时，认证链路会启用 Redis 驱动的登录态失效层： + - access token 按 `userId + clientType` 记录“在此时间点之前签发的 token 失效”。 + - refresh token 按 hash 写入黑名单，TTL 与剩余有效期对齐。 +- `POST /api/auth/login`、`POST /api/auth/register`、`POST /api/auth/dev-login`：如果是同客户端重新签发登录态，旧 access token 会被写入 Redis 失效层，并继续保留原有 `sid` 会话匹配语义。 +- `POST /api/user/password`、管理员封禁/改密/重置密码相关路径：会同时触发 access token Redis 失效标记与数据库 refresh token 撤销。 +- `POST /api/auth/refresh`：旧 refresh token 在数据库撤销之外，还会同步写入 Redis 黑名单；先命中黑名单的 token 会被直接拒绝。 +- 当 Redis 关闭时，系统会自动回退到原有的数据库 refresh token + `sid` 会话校验语义，不影响本地与 dev 启动。 +## 2026-04-10 Redis Files Cache And Upload Runtime + +- `GET /api/files/list` + - 对外语义不变，仍使用 `path`、`page`、`size` 参数返回当前用户目录分页结果。 + - 当 `app.redis.enabled=true` 时，后端会把热点目录页写入 Redis `files:list` cache，并通过目录版本号在创建、删除、移动、复制、重命名、恢复、上传完成和导入后做精准失效。 + - 搜索结果、回收站列表和后台任务列表不复用这套 key，避免不同语义的分页结果互相污染。 + +- `GET /api/v2/files/upload-sessions/{sessionId}` + - 响应体新增 `runtime` 字段；当 Redis 运行态存在时返回实时上传快照，不存在时返回 `null`，不影响原有会话元数据字段。 + - `runtime` 当前包含 `phase`、`uploadedBytes`、`uploadedPartCount`、`progressPercent`、`lastUpdatedAt`、`expiresAt`。 + - 该运行态由后端在会话创建、分片记录、代理上传、完成、取消、失败和过期时刷新，属于短生命周期缓存，不替代数据库里的最终状态。 + +- `POST /api/files/recycle-bin/{fileId}/restore` + - 外部接口不变，但 Redis 启用时后端会为同一 `fileId` 的恢复流程加分布式锁，避免多实例或并发请求重复恢复同一批条目。 + +## 2026-04-10 Redis Lightweight Broker First Landing + +- 本批次没有新增对外 HTTP API；用户可见接口仍沿用现有 `/api/files/**` 与 `/api/v2/tasks/**`。 +- 媒体文件通过网盘主链路落库后，后端现在会在事务提交后向轻量 broker 发布一次 `media-metadata-trigger`。这条触发只用于异步创建后台任务，不直接暴露为额外接口。 +- broker 当前只承载“自动补一条 `MEDIA_META` 任务”这一类轻量异步触发，最终执行状态、重试与公开结果仍以 `BackgroundTask` 记录和 `/api/v2/tasks/**` 查询结果为准。 +- `POST /api/v2/tasks/media-metadata` + - 用户手动创建任务的接口语义不变。 + - 与此同时，媒体文件成功落库后也可能由后端自动补一条同类任务；系统会按 `correlationId` 去重，避免同一文件被 broker 重复创建多条自动任务。 + +## 2026-04-10 Redis Transfer Session Store + +- 本批次没有新增快传 HTTP API，`/api/transfer/sessions`、`/api/transfer/sessions/lookup`、`/api/transfer/sessions/{sessionId}/join` 与信令轮询接口的对外协议保持不变。 +- 当 `app.redis.enabled=true` 时，在线快传 session 会写入 Redis `transfer-sessions` 命名空间，而不再只保存在当前 JVM 进程内；这让 `lookup/join/postSignal/pollSignals` 在多实例部署下具备共享同一在线会话状态的基础。 +- session 数据在 Redis 中会同时保存： + - `session:{sessionId}`：完整在线快传运行态快照。 + - `pickup:{pickupCode}`：`pickupCode -> sessionId` 映射。 +- Redis 关闭时，系统会自动回退到原有进程内存 store，本地和 dev 环境不需要额外 Redis 也能继续运行。 +- 离线快传不走这套 Redis store，仍继续使用数据库 `OfflineTransferSession` 持久化模型。 +## 2026-04-10 Redis File Event Pub/Sub + +- `GET /api/v2/files/events?path=/` + - 对外 SSE 协议不变，仍要求登录并支持 `X-Yoyuzh-Client-Id`。 + - 首次连接仍先收到 `READY` 事件，订阅路径过滤和同 `clientId` 自抑制语义保持不变。 + - 当 `app.redis.enabled=true` 时，某个实例在事务提交后写入的文件事件会额外通过 Redis pub/sub 广播到其他实例，因此同一用户连到不同后端实例时也能收到变更通知。 + - Redis pub/sub 只传播最小事件快照，不传播 `SseEmitter`、不重写 `FileEvent` 表，也不改变 `FileEvent` 作为审计持久化记录的角色。 + - Redis 关闭时会自动回退为原有单实例本地广播行为。 +## 2026-04-10 Spring Cache Minimal Landing + +- `GET /api/admin/storage-policies` + - 瀵瑰鍗忚涓嶅彉銆? + - 褰?`app.redis.enabled=true` 鏃讹紝鍚庣浼氬皢鏁翠釜瀛樺偍绛栫暐鍒楄〃缂撳瓨鍒?`admin:storage-policies`銆? + - 褰?POST/PUT/PATCH` 瀛樺偍绛栫暐绠＄悊鎺ュ彛鍐欏叆鎴愬姛鍚庯紝缂撳瓨浼氱珛鍗宠澶辨晥锛屽悗缁璇锋眰浼氶噸寤烘柊鍒楄〃銆? + +- `GET /api/app/android/latest` + - 瀵瑰鍗忚涓嶅彉锛屼粛鏄叕寮€鎺ュ彛銆? + - 褰?`app.redis.enabled=true` 鏃讹紝鍚庣浼氬皢浠?`android/releases/latest.json` 鏋勫缓鍑虹殑 release metadata 鍝嶅簲缂撳瓨鍒?`android:release`銆? + - 杩欎釜缂撳瓨褰撳墠渚濊禆 TTL 鍒锋柊锛屽洜涓?latest metadata 鐨勬洿鏂版潵鑷?Android 鍙戝竷鑴氭湰鍐欏叆瀵硅薄瀛樺偍锛岃€屼笉鏄悗绔唴閮ㄦ煇涓鐞嗗啓鎺ュ彛銆? + +- `GET /api/admin/summary` + - 褰撳墠鏆備笉鎺ュ叆 Spring Cache銆? + - 鍘熷洜鏄繖涓?summary 鍚屾椂鍚湁 request count銆乧aily active users銆乭ourly timeline 绛夐珮棰戠粺璁″€硷紝鐢ㄦ樉寮忓け鏁堝緢闅惧湪褰撳墠鏋舵瀯涓嬩繚鎸佸共鍑€璇箟銆? +## 2026-04-10 Spring Cache Minimal Landing Clarification + +- `GET /api/admin/storage-policies` + - Response shape is unchanged. + - When `app.redis.enabled=true`, the backend caches the full storage policy list in `admin:storage-policies`. + - Successful storage policy create, update, and status-change writes evict that cache immediately. + +- `GET /api/app/android/latest` + - Response shape is unchanged. + - When `app.redis.enabled=true`, the backend caches the metadata response derived from `android/releases/latest.json` in `android:release`. + - Refresh is TTL-based because the metadata is updated by the Android release publish script rather than an in-app admin write endpoint. + +- `GET /api/admin/summary` + - This endpoint is intentionally not cached at the moment. + - The response mixes high-churn metrics such as request count, daily active users, and hourly request timeline data, so there is not yet a clean explicit invalidation boundary. +## 2026-04-10 DogeCloud Temporary S3 Session Clarification + +- No HTTP API contract changed in this batch. +- The decision for Step 11 is architectural: DogeCloud temporary S3 sessions remain cached per backend instance inside `DogeCloudS3SessionProvider`. +- This does not change upload, download, direct-upload, or multipart endpoint shapes; it only clarifies that cross-instance Redis reuse is intentionally not introduced for these temporary runtime sessions. +## 2026-04-10 Stage 1 Validation Clarification + +- No API response shape changed in Step 12. +- Validation confirmed that all new Redis-backed integrations added in Stage 1 still preserve the existing no-Redis API startup path when `app.redis.enabled=false`. +- Local boot also confirmed that the backend now has one explicit non-Redis prerequisite for runtime startup in both default and `dev` profiles: `app.jwt.secret` must be configured via `APP_JWT_SECRET` and cannot be left empty. +- Cross-instance behavior described by earlier Stage 1 notes remains architecturally valid, but it still needs real-environment verification with Redis plus multiple backend instances before being treated as deployment-proven. + +## 2026-04-10 Manual Redis Validation Addendum + +- No HTTP endpoint shape changed in this addendum either. +- The local two-instance Redis validation did confirm these existing API behaviors in a real runtime flow: +- `POST /api/auth/dev-login` on one instance invalidates the prior access token and refresh token even when the next authenticated read happens on the peer instance. +- `POST /api/transfer/sessions` plus `GET /api/transfer/sessions/lookup` continue to work across instances for online sessions, including after the creating instance is stopped. +- `GET /api/v2/files/events` on instance B receives a `CREATED` event after an authenticated media upload to instance A. +- `GET /api/v2/tasks` on instance B exposes the queued `MEDIA_META` task auto-created by that upload. +- Three backend fixes were internal and did not change API contracts: +- Redis cache serialization/deserialization for file list pages; +- Redis auth revocation cutoff precision; +- non-null `storage_name` persistence for directory creation and normal file upload metadata. + +## 2026-04-11 Admin Backend Surface Addendum + +- `GET /api/admin/file-blobs` + - Auth: admin only. + - Query params: `page`, `size`, `userQuery`, `storagePolicyId`, `objectKey`, `entityType`. + - Response items expose `FileEntity`-centric blob inspection fields including `objectKey`, `entityType`, `storagePolicyId`, `referenceCount`, `linkedStoredFileCount`, `linkedOwnerCount`, `sampleOwnerUsername`, `sampleOwnerEmail`, `createdByUserId`, `createdByUsername`, `blobMissing`, `orphanRisk`, and `referenceMismatch`. + - This endpoint is for admin diagnostics and migration visibility; it does not replace end-user file reads. + +- `GET /api/admin/shares` + - Auth: admin only. + - Query params: `page`, `size`, `userQuery`, `fileName`, `token`, `passwordProtected`, `expired`. + - Response items expose share metadata from `FileShareLink`, plus owner and file summary fields. + +- `DELETE /api/admin/shares/{shareId}` + - Auth: admin only. + - Deletes the target `FileShareLink` immediately. + - Intended for operational cleanup and moderation. + +- `GET /api/admin/tasks` + - Auth: admin only. + - Query params: `page`, `size`, `userQuery`, `type`, `status`, `failureCategory`, `leaseState`. + - Response items expose task owner identity plus parsed task-state helpers: `failureCategory`, `retryScheduled`, `workerOwner`, and derived `leaseState`. + +- `GET /api/admin/tasks/{taskId}` + - Auth: admin only. + - Returns the same admin task response shape as the list endpoint for a single task. diff --git a/docs/architecture.md b/docs/architecture.md index bc4843a..ebcf120 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -514,3 +514,132 @@ Android 壳补充说明： - worker 现在会按失败分类和任务类型做自动重试：失败会归到 `UNSUPPORTED_INPUT`、`DATA_STATE`、`TRANSIENT_INFRASTRUCTURE`、`RATE_LIMITED`、`UNKNOWN`；其中 `ARCHIVE` 默认最多 4 次、`EXTRACT` 最多 3 次、`MEDIA_META` 最多 2 次，公开 state 会暴露 `attemptCount/maxAttempts/retryScheduled/nextRetryAt/retryDelaySeconds/lastFailureMessage/lastFailureAt/failureCategory`。 - 当前仍不包含非 zip 解压格式、缩略图/视频时长任务，以及 archive/extract 的前端入口。 - 桌面端 `front/src/pages/Files.tsx` 已接入最近 10 条后台任务查看与取消入口，并可为当前选中文件创建 `MEDIA_META` 任务；移动端与 archive/extract 的前端入口仍未接入。 +## 11. UI 视觉系统与主题引擎 (2026-04-10 升级) + +### 11.1 设计语言：Stitch Glassmorphism + +全站视觉系统已全面转向“Stitch”玻璃拟态 (Glassmorphism) 风格，其核心特征包括： + +- **全局背景 (Aurora)**：在 `index.css` 中定义了 `bg-aurora`，结合颜色渐变与动态光晕产生深邃的底色。 +- **玻璃面板 (.glass-panel)**：核心 UI 容器均使用半透明背景 (`bg-white/40` 或 `bg-black/40`)、高饱和背景模糊 (`backdrop-blur-xl`) 和细腻的白色线条边框 (`border-white/20`)。 +- **浮动质感**：通过 `rounded-3xl` 或 `rounded-[2rem]` 的大圆角和外阴影增强层叠感。 + +### 11.2 主题管理 (Theme Engine) + +系统内建了一套完整的主题上下文，主要路径为： + +- `front/src/components/ThemeProvider.tsx`：提供 `light | dark | system` 主题状态切换与持久化，通过操作 `html` 根节点的 `class` 实现。 +- `front/src/components/ThemeToggle.tsx`：全局主题切换按钮组件。 +- `front/src/lib/utils.ts`：提供 `cn()` 工具函数，用于处理 Tailwind 类的动态组合与主题适配。 + +### 11.3 模块适配情况 + +- **用户侧**：网盘、快传、分享详情、任务列表、回收站均已完成适配。所有表格、卡片和导航栏均已升级为玻璃态。 +- **移动端**：`MobileLayout` 实现了一套悬浮式玻璃顶部标题栏与底部导航栏，并保持与桌面端一致的光晕背景。 +- **管理侧**：Dashboard 大盘指标卡片、用户列表、文件审计列表和存储策略列表均已同步升级。 + +## 12. Redis Foundation (2026-04-10) + +- 后端已引入 Spring Data Redis 与 Spring Cache，但 Redis 仍是可选基础设施：`app.redis.enabled=false` 时，应用会回退到 no-op token 失效服务与 `NoOpCacheManager`，本地与 dev 环境不需要外部 Redis 也能正常启动与测试。 +- Redis 配置拆成两层： + - `spring.data.redis.*`：连接参数。 + - `app.redis.*`：业务 key prefix、TTL buffer、cache TTL 与命名空间。 +- 当前声明的 Redis 命名空间包括：`cache`、`auth`、`transfer-sessions`、`upload-state`、`locks`、`file-events`、`broker`。本轮真正落地使用的是 `auth`，其余属于后续 Stage 1 边界预留。 +- 当前声明的 Spring Cache 名称包括：`files:list`、`admin:summary`、`admin:storage-policies`、`android:release`。本轮只完成了缓存边界与 TTL 骨架，尚未把具体读路径接到这些 cache。 +- 认证链路新增 Redis 失效层： + - access token：按 `userId + clientType` 记录“在此时间点之前签发的 token 失效”。 + - refresh token：按 token hash 写入黑名单，TTL 与剩余有效期对齐。 +- `JwtAuthenticationFilter` 现在会先检查 access token 是否已被 Redis 失效层拒绝，再继续执行原有的 JWT 校验、用户加载与 `sid` 会话匹配。 +- `AuthService` 与 `AdminService` 的同端重登、改密、封禁、管理员重置密码路径，现已统一调用这层服务；`RefreshTokenService` 在轮换、过期拒绝与批量撤销时也会同步刷新 refresh token 黑名单。 +## 12.1 Redis Foundation Batch 2 (2026-04-10) + +- `FileService.list(...)` 现已通过 `FileListDirectoryCacheService` 接入可选 Redis 热目录缓存，当前只缓存 `/api/files/list` 的目录分页结果，不混入搜索、回收站或后台任务列表。 +- 热目录缓存使用 `files:list` Spring Cache 命名空间，真实缓存 key 由 `userId + normalized path + page + size + fixed sort context + directory version` 组成；目录版本存放在 Redis KV 中，按目录粒度增量失效，避免全局清空。 +- 目录列表失效点已经覆盖 `mkdir`、上传完成、外部导入、回收站删除、回收站恢复、重命名、移动、复制与默认目录补齐，所有变更最终都归一到 `touchDirectoryListings(...)`。 +- 分布式锁新增 `DistributedLockService` 抽象与 Redis 实现，当前第一批只落在 `FileService.restoreFromRecycleBin(...)`，锁 key 为 `files:recycle-restore:{fileId}`，通过 `SETNX + TTL + owner token` 获取并用 Lua compare-and-delete 释放。 +- 上传会话运行态新增 `UploadSessionRuntimeStateService` 抽象与 Redis 实现，短生命周期状态写入 `upload-state` 命名空间；数据库里的 `UploadSession` 继续承担最终事实，Redis 只承载创建中、上传中、完成中这类运行态快照。 +- `UploadSessionV2Controller` 已把运行态映射到响应体 `runtime` 字段，便于前端轮询时直接读取 phase、已上传字节数、分片数、进度百分比与过期时间，而不需要额外拼装临时状态。 + +## 12.2 Redis Foundation Batch 3 (2026-04-10) + +- 轻量 broker 已新增 `LightweightBrokerService` 抽象：Redis 启用时使用 `RedisLightweightBrokerService` 把消息写入 Redis list；Redis 关闭时回退到 `InMemoryLightweightBrokerService`，继续支持本地单实例开发与测试。 +- 这层 broker 明确只服务“小规模、低成本、可接受保守语义”的异步触发，不承担高可靠消息系统职责；任务最终状态、重试、幂等与用户可见结果仍以数据库 `BackgroundTask` 为准。 +- 当前 broker 使用 `app.redis.namespaces.broker` 命名空间，首个 topic 为 `media-metadata-trigger`，消息体只携带最小触发上下文：`userId`、`fileId`、`correlationId`。 +- `FileService.saveFileMetadata(...)` 现在会在媒体文件元数据落库后通过 `MediaMetadataTaskBrokerPublisher` 做 after-commit 发布；非媒体文件、目录、缺少必要主键信息的条目不会进入 broker。 +- `MediaMetadataTaskBrokerConsumer` 通过定时 drain 方式消费 broker 消息，并调用 `BackgroundTaskService.createQueuedAutoMediaMetadataTask(...)` 创建 `MEDIA_META` 任务；该入口会先按 `correlationId` 去重，再校验文件仍存在、未删除且仍属于媒体文件，避免重复建任务。 +- 这批实现的目标是“让轻量 broker 先承担一类真实异步触发”，而不是替代现有 `BackgroundTask` worker，也不覆盖文件事件跨实例广播；后者仍归 Stage 1 Step 9 处理。 + +## 12.3 Redis Foundation Batch 4 (2026-04-10) + +- 在线快传 session 已从进程内 `ConcurrentHashMap` 提升为可选 Redis 支撑：`TransferSessionStore` 在 Redis 启用时把 session JSON 与 `pickupCode -> sessionId` 映射写入 `transfer-sessions` 命名空间，关闭时自动回退到原有内存模式。 +- Redis key 当前按 `session:{sessionId}` 与 `pickup:{pickupCode}` 组织，TTL 与 session `expiresAt` 对齐并附带 `app.redis.ttlBufferSeconds` 缓冲；因此 Redis 只承载在线快传的短生命周期运行态，不替代离线快传数据库模型。 +- `TransferSession` 新增内部快照序列化形状，用于保留 `receiverJoined`、信令队列、cursor 和文件清单等运行期状态；`joinSession`、`postSignal` 在修改在线 session 后会重新写回 store，避免 Redis 模式下只改内存副本而不持久化。 +- `TransferService.nextPickupCode()` 现在复用 `TransferSessionStore.nextPickupCode()`；Redis 启用时 pickup code 会先在 Redis 映射 key 上做短 TTL 预留，降低多实例并发创建在线快传 session 时的冲突概率。 +- 当前 Step 8 只覆盖在线快传 session 的跨实例 lookup/join 基础能力；离线快传仍继续使用 `OfflineTransferSessionRepository`，文件事件广播也仍留在 Step 9。 +## 12.4 Redis Foundation Batch 5 (2026-04-10) + +- 文件事件跨实例分发现在落地在 Redis pub/sub，而不是把 `SseEmitter` 或订阅状态搬进 Redis。每个实例仍只在本地维护 `userId -> subscriptions` 的内存映射，SSE 过滤逻辑继续由 `FileEventService` 负责。 +- `FileEventService.record(...)` 现在仍然先写 `FileEvent` 表；事务提交后会先向本实例订阅者投递，再通过 `FileEventCrossInstancePublisher` 把最小事件快照发布到 `keyPrefix:file-events:pubsub` topic。 +- Redis 开启时，`RedisFileEventPubSubPublisher` 会附带当前实例 `instanceId`；`RedisFileEventPubSubListener` 在收到消息后会忽略同实例回环消息，只把远端事件重建成 `FileEvent` 并交回 `FileEventService.broadcastReplicatedEvent(...)` 做本地 SSE 投递。 +- 这条链路的目标是“跨实例转发已提交的文件事件”，不是高可靠消息系统：它不重放历史事件，不替代 `FileEvent` 表持久化，也不承担断线补偿；真正的事件审计事实源仍然是数据库。 +- Redis 关闭时，`NoOpFileEventCrossInstancePublisher` 会让行为自动回退为原有单实例本地广播，dev 与本地测试环境不需要额外 Redis 也能继续运行。 +## 12.5 Redis Foundation Batch 6 (2026-04-10) + +- Spring Cache 鍦ㄨ繖涓€鎵规寮忔帴鍏ヤ簡涓ょ被楂樿浣庡啓璇昏矾寰勶細`AdminService.listStoragePolicies()` 浣跨敤 `admin:storage-policies`锛宍AndroidReleaseService.getLatestRelease()` 浣跨敤 `android:release`銆? +- 瀛樺偍绛栫暐鍒楄〃鐨勭紦瀛樺け鏁堢偣鏄槑纭殑绠＄悊鍐欒矾寰勶細鍒涘缓銆佺紪杈戙€佸惎鍋滈兘鍦?`AdminService` 涓婄洿鎺?evict锛屼笉鎶婂叾浠栫敤鎴疯矾寰勬垨鏂囦欢璇昏矾寰勬贩杩涘悓涓€ cache銆? +- Android release metadata 鍒欐槸 TTL 椹卞姩鐨勭紦瀛橈細鏁版嵁婧愪粛鏄璞″瓨鍌ㄧ殑 `android/releases/latest.json`锛屽悗绔彧缂撳瓨鏋勫缓鍚庣殑 `AndroidReleaseResponse`锛屼笉缂撳瓨 APK 鍒嗗彂瀛楄妭娴併€? +- `admin summary` 缁忚瘎浼板悗鏆備笉鎺ュ叆缂撳瓨锛屽洜涓鸿繖涓?DTO 鍚屾椂缁勫悎浜嗛珮棰戝彉鍖栫殑 request metrics銆佹瘡鏃ユ椿璺冪敤鎴风粺璁″拰閭€璇风爜绛夊€硷紝鐩墠娌℃湁涓€涓共鍑€鐨勬樉寮忓け鏁堣竟鐣岄€傚悎鎶婂畠鏀惧叆 Spring Cache銆? +## 12.5 Redis Foundation Batch 6 Clarification (2026-04-10) + +- Spring Cache is now active on two high-read, low-write backend read paths. +- `AdminService.listStoragePolicies()` uses cache `admin:storage-policies`. +- `AndroidReleaseService.getLatestRelease()` uses cache `android:release`. +- Storage policy cache invalidation is explicit and tied to admin create, update, and status-change writes. +- Android release metadata uses TTL-based refresh because the source of truth is object storage metadata at `android/releases/latest.json`, updated by the release publish script rather than an in-app write path. +- APK byte streaming remains uncached; only the metadata response is cached. +- `admin summary` remains uncached by design because it mixes several high-churn metrics and does not yet have a clean invalidation boundary. +## 12.6 Redis Foundation Batch 7 Clarification (2026-04-10) + +- `DogeCloudS3SessionProvider` intentionally remains a per-instance in-memory cache instead of moving to Redis. +- The cached object is not just raw temporary credentials; it is a live runtime session containing `S3Client` and `S3Presigner`, both of which have local lifecycle and cleanup semantics. +- Because of that, a Redis-backed shared cache would either have to cache only raw credential material and rebuild SDK clients locally anyway, or attempt to share values that are not meaningful across JVM instances. +- The current design keeps refresh ownership local to each backend instance: if cached credentials are still outside the one-minute refresh window, the existing runtime session is reused; once inside that window, the old runtime session is closed and a fresh one is fetched and rebuilt. +- This leaves some duplicate DogeCloud temporary-token fetches in multi-instance deployments, but the current plan judges that cost lower than the added complexity and secret-handling surface of a Redis shared-credential cache. +## 12.7 Redis Foundation Batch 8 Clarification (2026-04-10) + +- Stage 1 validation closed with two local checks: full backend test regression and a Redis-disabled `dev` boot-path check. +- The local boot-path check matters because Redis integration is optional by design. With `APP_REDIS_ENABLED=false`, the application still starts as a normal single-instance backend once mandatory base config such as `APP_JWT_SECRET` is present. +- In the validated local path, the backend started successfully on an alternate port (`18081`) under the `dev` profile, using H2 and no Redis dependency. +- Therefore the current architecture boundary remains unchanged: Redis augments cache, pub/sub, lock, broker, and short-lived runtime state when enabled, but it is not a required baseline component for local development or single-instance fallback. +- The architecture still has explicit environment-bound gaps that were not closed in-process: real Redis reliability/TTL observation and cross-instance propagation timing for file events, lightweight broker delivery, upload runtime state, and transfer-session sharing. + +## 12.8 Manual Redis Validation Clarification (2026-04-10) + +- The later manual validation pass did exercise real local Redis plus two backend instances, so several Stage 1 architecture claims are now locally runtime-validated rather than only unit/integration-tested. +- Verified runtime behaviors: +- auth token invalidation survives cross-instance login churn; +- online transfer runtime state survives loss of the creating instance; +- file events can cross instances through the SSE path when a real uploaded file triggers a `CREATED` event; +- the lightweight broker can auto-create a queued `MEDIA_META` task after a media upload and that task is visible from the peer instance. +- The Redis file list cache architecture also needed one implementation detail clarified: Spring Cache may hand back generic decoded maps from Redis, so `RedisFileListDirectoryCacheService` now treats cache-value reconstruction as an application concern instead of assuming a strongly typed cache provider result. +- The persistence model also still carries `portal_file.storage_name` as a required column in the live schema, so even after blob/entity migration work the backend must continue writing a non-null legacy storage name for directories and uploaded files until a later schema migration explicitly removes that requirement. +- One environment gap remains: local `redis-cli` key inspection did not reveal the expected keys during probing even while cross-instance behavior proved shared runtime state was active. That means the current architectural confidence comes from observable runtime behavior, not from direct local key-space inspection. + +## Debugging Discipline + +- Use short bounded probes first when validating network, dependency, or startup issues. Prefer commands such as `curl --max-time`, `mvn -q`, `mvn dependency:get`, `apt-get update`, and similar narrow checks before launching long-running downloads or full test runs. +- Do not wait indefinitely on a stalled download or progress indicator. If a command appears stuck, stop and re-check DNS, proxy inheritance, mirror reachability, and direct-vs-proxy routing before retrying. +- For WSL debugging, verify the proxy path and the direct path separately, then choose the shortest working route. Do not assume a mirror problem until the network path has been isolated. +- Use domestic mirrors as a delivery optimization, not as a substitute for diagnosis. First determine whether the failure is caused by DNS, proxy configuration, upstream availability, or the mirror itself. + +## 12.9 Admin Backend Surface Clarification (2026-04-11) + +- The admin module now covers four distinct backend inspection domains: +- user and summary management; +- logical file management; +- storage policy management and migration task creation; +- operational inspection for file blobs, shares, and background tasks. +- `GET /api/admin/file-blobs` is architected around `FileEntity` plus `StoredFileEntity` relations instead of around `StoredFile` rows. This keeps the admin surface aligned with the newer object/entity model and lets operators inspect storage-policy ownership, reference counts, and missing-object anomalies before the legacy read path is retired. +- `GET /api/admin/shares` and `DELETE /api/admin/shares/{shareId}` sit on top of `FileShareLinkRepository` and are intended as operational controls for share hygiene rather than end-user sharing flows. +- `GET /api/admin/tasks` and `GET /api/admin/tasks/{taskId}` sit on top of `BackgroundTaskRepository` and parse structured fields out of `publicStateJson` so the admin UI can inspect failure category, retry scheduling, worker owner, and lease freshness without re-implementing backend parsing rules. +- This batch does not change the current production read-path boundary: download, share detail, recycle-bin, and zip flows still read from `StoredFile.blob`, while `FileEntity` and `StoredFile.primaryEntity` continue to carry migration-oriented metadata for newer admin and storage-policy workflows. diff --git a/docs/superpowers/plans/2026-04-10-cloudreve-gap-next-phase-upgrade.md b/docs/superpowers/plans/2026-04-10-cloudreve-gap-next-phase-upgrade.md new file mode 100644 index 0000000..1dca7d6 --- /dev/null +++ b/docs/superpowers/plans/2026-04-10-cloudreve-gap-next-phase-upgrade.md @@ -0,0 +1,712 @@ +# Cloudreve Gap Next-Phase Upgrade Plan + +> **For agentic workers:** REQUIRED: Use `superpowers:executing-plans` or `superpowers:subagent-driven-development` when implementing this plan. Keep the checkbox state updated as work lands. + +**Goal:** 在不偏离当前产品方向的前提下，把项目中“对比 Cloudreve 仍明显缺失”的能力拆成可执行的后续升级阶段，优先补齐最能提升网盘完成度和平台化能力的部分，而不是盲目追平 Cloudreve 的全部生态。 + +**Repository:** `C:\Users\yoyuz\Documents\code\my_site` + +**Decision:** 当前项目已经完成 v2 上传会话、存储策略、分享二期、搜索骨架、SSE 文件事件、后台任务骨架、回收站、Android 壳和前后端视觉重构。后续计划只覆盖“尚未完成且仍值得做”的能力，不重复规划已经落地的阶段。 + +## 1. Current Baseline + +下列能力已完成，不应重新当作“待做”： + +- v2 upload session 已支持 `PROXY` / `DIRECT_SINGLE` / `DIRECT_MULTIPART` +- 存储策略管理、迁移任务和策略能力声明已落地 +- 分享二期、文件搜索、文件事件 SSE、后台任务框架已落地 +- 回收站、媒体元数据任务、桌面端任务面板已落地 +- 前后端 UI 已完成一次系统性重构 + +当前仍明确未完成或仅完成一半的点： + +- 后端尚未接入 Redis；当前没有 Spring Cache，也没有跨实例缓存/会话总线 +- 移动端文件搜索未接入 +- `ARCHIVE` / `EXTRACT` 前端入口未接入，移动端任务入口也未接入 +- 旧下载/分享详情/ZIP/回收站读取路径仍依赖 `StoredFile.blob`，尚未切到 `primaryEntity` +- 仅有媒体元数据提取，没有缩略图、视频时长、预览资源管线 +- 没有 WebDAV +- 没有远程离线下载器能力 +- 没有 OIDC / OAuth scope / 桌面同步客户端协议 +- 不建议当前阶段直接做完整 WOPI / Office 在线协作 + +## 2. Scope And Priority + +### P0: 先补齐现有平台里的断点 + +这部分不做新产品线，只把已经有骨架但没闭环的能力补完整： + +1. Redis 基础接入与缓存边界落地 +2. 移动端搜索接入 +3. `ARCHIVE` / `EXTRACT` 前端入口 +4. 移动端任务入口 +5. 旧读取路径从 `StoredFile.blob` 迁到 `primaryEntity` + +### Admin Console Alignment + +参考成熟项目的后台目录，当前项目后续管理台不应只停留在 `dashboard / users / files / storage-policies` 四类资源，而应逐步演进为以下信息架构： + +1. 面板首页 +2. 参数设置 +3. 文件系统 +4. 存储策略 +5. 节点 +6. 用户组 +7. 用户 +8. 文件 +9. 文件 Blob +10. 分享 +11. 后台任务 +12. 订单 +13. 事件 +14. 滥用举报 +15. OAuth 应用 + +其中适合当前项目近期推进的只有： + +- 参数设置 +- 文件系统 +- 存储策略 +- 用户 +- 文件 +- 文件 Blob +- 分享 +- 后台任务 +- OAuth 应用（先预留，不急于完整实现） + +当前阶段明确延后： + +- 节点 +- 用户组 +- 订单 +- 事件独立审计中心 +- 滥用举报 + +### P1: 预览与媒体管线 + +这是最值得补的 Cloudreve 差距： + +1. 图片缩略图 +2. 视频 poster / 时长 +3. 文件列表与详情中的预览消费 +4. 失败状态与 metadata 持久化 + +### P2: WebDAV 最小可用版 + +只做单用户私有网盘最小读写，不提前做复杂共享挂载、锁协商和第三方 scope。 + +### P3: 生态扩展 backlog + +这部分先保留为后续阶段，不在最近一轮升级中直接开工： + +1. 远程离线下载 +2. OIDC / OAuth scope +3. 桌面同步客户端协议 +4. WOPI / Office 在线协作 + +## 3. Non-Goals + +- 不把项目改造成 Cloudreve 克隆 +- 不新增与当前业务方向不一致的组织/团队协作大系统 +- 不在当前阶段接入完整 WOPI +- 不为了“对齐功能表”而重做现有快传业务 +- 不引入仓库中不存在的验证命令 + +--- + +## 4. Stage 1: Redis Cache Foundation + +**Goal:** 为后端引入 Redis，先解决真正适合走缓存或跨实例共享的状态，不把所有内存结构机械搬过去。 + +**Why now:** + +- 当前后端没有 `spring-boot-starter-data-redis` +- `TransferSessionStore` 仍是进程内 `ConcurrentHashMap` +- `FileEventService` 的订阅和广播只在单实例内有效 +- `DogeCloudS3SessionProvider` 只有本机进程内临时会话缓存 +- 你已经明确希望 Redis 承担登录态 / token 黑名单、热门目录缓存、分布式锁、上传状态缓存和小规模队列 broker + +**Files likely involved:** + +- `backend/pom.xml` +- `backend/src/main/resources/application.yml` +- `backend/src/main/resources/application-dev.yml` +- `backend/src/main/java/com/yoyuzh/config/*` +- `backend/src/main/java/com/yoyuzh/transfer/*` +- `backend/src/main/java/com/yoyuzh/files/events/*` +- `backend/src/main/java/com/yoyuzh/files/storage/*` +- `backend/src/test/java/com/yoyuzh/**` +- `docs/architecture.md` +- `docs/api-reference.md` +- `memory.md` + +- [x] **Step 1: 接入 Spring Data Redis 与配置骨架** + - 新增 Redis 依赖 + - 在 `application.yml` / `application-dev.yml` 加入 `spring.data.redis.*` 与 `app.redis.*` + - Redis 必须允许关闭，不强制 dev 环境依赖外部服务 + +- [x] **Step 2: 明确缓存分层，不做一锅炖** + - `Spring Cache`：用于热门目录和热点只读查询 + - `Redis KV`：用于登录态派生状态、token 黑名单、快传会话、上传状态 + - `Redis Lock`：用于分布式锁 + - `Redis Pub/Sub`：用于多实例文件事件分发 + - `Redis List/Stream` 或轻量队列表：用于小规模 broker + - 不把 JPA 实体直接无脑全量缓存 + +- [x] **Step 3: 接入登录态 / token 黑名单** + - 保持当前 JWT + refresh token 主模型不变 + - 新增 access token / refresh token 撤销或踢下线后的 Redis 黑名单能力 + - 黑名单 TTL 与 token 剩余有效期对齐，避免永久堆积 + - 用户改密、封禁、管理员重置密码、同端挤下线等场景统一走这套失效机制 + +- [x] **Step 4: 接入热门目录缓存** + - 优先缓存 `/api/files/list` 的热点目录页结果，而不是所有目录 + - 缓存 key 至少包含 `userId + path + page + size + sort context` + - 文件创建、删除、移动、重命名、恢复、导入、上传完成后精准失效相关目录 + - 不让搜索结果、回收站列表和任务列表混进同一套 key + +- [x] **Step 5: 接入分布式锁** + - 先覆盖会发生并发冲突或重复执行风险的路径 + - 优先考虑上传完成、存储策略迁移、后台任务 claim / retry、回收站恢复、目录批量导入 + - 锁必须带 TTL 和 owner 标识，避免死锁 + - 不用 Redis 锁去替代数据库事务 + +- [x] **Step 6: 接入上传状态缓存** + - 用于保存上传中的短生命周期状态，而不是取代数据库中的最终事实 + - 适合承载 chunk 进度、最近心跳、瞬时速度、前端轮询状态 + - `UploadSession` 仍保留数据库持久化和最终完成语义 + - Redis 状态过期后不应影响已完成或已失败的最终结果判断 + +- [x] **Step 7: 引入小规模队列 broker** + - 目标不是替代当前数据库任务系统，而是给轻量异步链路和跨实例触发提供 broker + - 优先承载文件事件转发、缩略图触发、媒体处理触发、低成本异步通知 + - 当前规模下可接受 Redis broker，但要明确“不是高可靠消息系统” + - 大任务最终状态仍以数据库 `BackgroundTask` 为准 + - 2026-04-10 首批落地先收敛到“媒体文件落库后的 `MEDIA_META` 自动触发”，文件事件跨实例广播仍留给 Step 9 的 Redis pub/sub + +- [x] **Step 8: 把 `TransferSessionStore` 改成 Redis 支撑的 session store** + - 替换当前本地 `ConcurrentHashMap` + - 保持过期清理和 pickup code 查询语义不变 + - 让在线快传在多实例下仍可 lookup/join + - 2026-04-10 当前实现为“Redis 启用时在线快传 session 走 Redis KV，关闭时自动回退到进程内存”，离线快传仍继续走数据库持久化链路 + +- [x] **Step 9: 接入文件事件跨实例分发** + - 保留当前单实例 emitter 管理 + - 新增 Redis pub/sub，把事务提交后的文件事件广播到其他实例 + - 避免把 `SseEmitter` 本身存进 Redis + +- [x] **Step 10: 评估并最小落地 Spring Cache** + - 优先考虑热门目录、`admin summary`、存储策略列表、Android 最新发布元数据等高读低写接口 + - 每个缓存都要有明确失效策略，不能只加 `@Cacheable` + +- [x] **Step 11: 审慎处理 DogeCloud 临时 S3 会话缓存** + - 若多实例下重复拉临时 token 成本可接受，则保留本地内存缓存 + - 若需要跨实例复用，再单独加 Redis 缓存，不与业务缓存混用 + +- [x] **Step 12: 验证** + - `cd backend && mvn test` + - 手动验证无 Redis 时应用仍可启动 + - 手动验证启用 Redis 后快传在线会话可创建、lookup、join、过期 + - 手动验证踢下线 / 改密后旧 token 失效 + - 手动验证热门目录缓存命中与目录变更后失效 + - 手动验证多实例下文件事件能跨实例到达 + - 手动验证任务重复 claim 不会发生明显并发冲突 + +**Exit criteria:** + +- Redis 成为可选但可用的基础设施 +- 登录态 / token 黑名单已接入 Redis +- 至少一个真实热点目录查询接入 Redis/Spring Cache +- 至少一个高风险并发路径接入分布式锁 +- 上传中的短生命周期状态已进入 Redis +- 小规模 broker 已承担至少一类轻量异步触发 +- 在线快传会话不再依赖单进程内存 +- 文件事件具备跨实例扩展边界 + +--- + +## Admin Track: Backend Management Surface + +**Goal:** 按更成熟的后台目录，把当前项目后端管理能力补成“资源可观测、可管理、可扩展”的体系，而不是继续把所有管理功能堆进单一 summary 页面。 + +### Admin-B1: Parameter Settings + +**Goal:** 新增“参数设置”资源，集中管理当前散落在配置和管理台中的系统开关。 + +**Recommended internal sections:** + +1. 站点信息 +2. 用户会话 +3. 验证码 +4. 媒体处理 +5. 增值服务 +6. 邮件 +7. 队列 +8. 外观 +9. 事件 +10. 服务器 + +**Current-project recommendation:** + +- 近期应实现： + - 站点信息 + - 用户会话 + - 媒体处理 + - 队列 + - 外观 + - 服务器 +- 可先留空壳或只读： + - 验证码 + - 邮件 + - 事件 +- 当前不建议投入： + - 增值服务 + +**Suggested scope:** + +- 注册与邀请策略 +- 离线快传总上限 +- 默认上传大小限制 +- 站点显示参数 +- 媒体处理开关 +- Redis / runtime 只读状态 + +- [ ] **Step 1: 设计参数设置 DTO 与权限边界** +- [ ] **Step 2: 先拆站点信息子分组** + - 站点名称 + - 站点描述 + - 主站点 URL + - 备用站点 URL + - 页脚代码 + - 登录公告 / 站点公告 + - 使用条款 / 隐私政策链接 +- [ ] **Step 3: 设计用户会话子分组** + - access / refresh 生命周期 + - 同端挤下线策略 + - token 黑名单开关与 TTL 策略 + - 登录安全相关策略 +- [ ] **Step 4: 设计媒体处理子分组** + - 媒体元数据提取开关 + - 缩略图开关 + - 视频 poster / 时长提取策略 + - 第三方依赖状态只读信息 +- [ ] **Step 5: 设计队列子分组** + - broker 类型 + - worker 并发 + - 失败重试预算 + - 队列健康状态只读信息 +- [ ] **Step 6: 设计外观与服务器子分组** + - 前端品牌化字段 + - CDN / 静态资源缓存参数 + - 服务器运行信息、Redis 状态、存储后端状态 +- [ ] **Step 2: 暴露管理员参数读取与更新接口** +- [ ] **Step 3: 只允许修改当前可安全热更新的参数** +- [ ] **Step 4: 文档化哪些配置仍需环境变量或重启** + +### Admin-B2: File System + +**Goal:** 把“文件系统”作为独立后台资源，而不是只在文件列表里做删除。 + +**Recommended internal sections:** + +1. 参数设置 +2. 全文搜索 +3. 文件图标 +4. 文件浏览应用 +5. 自定义属性 + +**Current-project recommendation:** + +- 近期应实现： + - 参数设置 + - 文件图标 + - 自定义属性 +- 可先做只读骨架： + - 文件浏览应用 +- 当前延后： + - 全文搜索 + +**Suggested scope:** + +- 默认存储后端概览 +- 上传模式能力矩阵 +- 媒体处理能力状态 +- 热门目录 / 缓存状态概览 +- WebDAV 预留状态 + +- [ ] **Step 1: 设计文件系统只读总览接口** +- [ ] **Step 2: 设计文件系统参数设置子分组** + - 文档在线编辑最大大小 + - 回收站扫描间隔 + - 文件 Blob 回收间隔 + - 静态资源缓存 TTL + - 文件列表分页方式 + - 最大分页大小 + - 最大批量操作数量 + - 最大递归搜索数量 + - 地图提供商 +- [ ] **Step 3: 设计文件图标子分组** + - 扩展名到图标的映射策略 + - 前端图标主题扩展点 + - 自定义 mime/icon 映射入口 +- [ ] **Step 4: 设计文件浏览应用子分组** + - 当前阶段只读显示已接入的浏览/预览能力 + - 后续为 WOPI / Office / 媒体浏览器预留入口 +- [ ] **Step 5: 设计自定义属性子分组** + - 标签 schema + - metadata key 命名约束 + - 可搜索属性白名单 +- [ ] **Step 6: 暴露上传模式、缓存、媒体处理、WebDAV 状态** +- [ ] **Step 7: 管理台可按模块查看文件系统运行态** + +### Admin-B3: File Blob + +**Goal:** 既然项目已经有 `FileBlob` / `FileEntity`，后台必须能查看物理对象层，而不是只看逻辑文件。 + +**Suggested scope:** + +- Blob / Entity 基本信息 +- object key +- storage policy +- reference count +- orphan 风险 +- 派生实体类型：`VERSION` / `THUMBNAIL` / `TRANSCODE` + +- [ ] **Step 1: 增加管理员 Blob/Entity 列表接口** +- [ ] **Step 2: 支持按用户、策略、对象 key、实体类型过滤** +- [ ] **Step 3: 标注高风险条目，如引用异常、迁移失败残留** + +### Admin-B4: Share + +**Goal:** 分享需要成为独立后台资源，便于管理滥用和过期内容。 + +- [ ] **Step 1: 管理员分享列表接口** +- [ ] **Step 2: 支持按用户、文件名、token、是否密码保护、是否过期过滤** +- [ ] **Step 3: 支持管理员撤销分享** + +### Admin-B5: Background Tasks + +**Goal:** 后台任务不能只在用户视角可见，管理员也要能看全局任务池。 + +- [ ] **Step 1: 增加管理员任务列表与详情接口** +- [ ] **Step 2: 支持按任务类型、状态、失败分类、租约状态过滤** +- [ ] **Step 3: 支持查看任务归属用户、重试信息、锁/worker 信息** + +### Admin-B6: OAuth Apps + +**Goal:** 为未来 WebDAV / 第三方客户端 / OIDC 留出后台资源位。 + +- [ ] **Step 1: 当前阶段只预留数据模型与只读列表边界** +- [ ] **Step 2: 不急于完整实现授权流程** +- [ ] **Step 3: 在文档中明确这是后续阶段入口** + +--- + +## 5. Stage 2: Close Existing v2 Gaps + +**Goal:** 把现有架构中的“半完成状态”补成真正可用的闭环，优先提升当前产品完成度。 + +**Files likely involved:** + +- `front/src/pages/Files.tsx` +- `front/src/mobile-pages/MobileFiles.tsx` +- `front/src/lib/file-search.ts` +- `front/src/lib/file-events.ts` +- `front/src/lib/upload-session.ts` +- `front/src/lib/api.ts` +- `front/src/mobile-components/*` +- `backend/src/main/java/com/yoyuzh/files/core/*` +- `backend/src/main/java/com/yoyuzh/files/tasks/*` +- `backend/src/main/java/com/yoyuzh/files/search/*` +- `backend/src/test/java/com/yoyuzh/files/**` +- `front/src/**/*.test.ts` + +- [ ] **Step 1: 移动端接入 v2 文件搜索** + - 复用现有 `front/src/lib/file-search.ts` + - 保持与桌面端相同的查询参数和空态行为 + - 不把搜索结果写回目录缓存 + +- [ ] **Step 2: 桌面端补齐 `ARCHIVE` / `EXTRACT` 入口** + - 从当前选中文件直接创建任务 + - 在任务面板中区分 `ARCHIVE`、`EXTRACT`、`MEDIA_META` + - 错误态展示后端返回的任务失败原因 + +- [ ] **Step 3: 移动端补齐任务入口** + - 至少支持查看最近任务 + - 支持取消 `QUEUED` / `RUNNING` + - 支持为选中文件创建 `MEDIA_META` + - 如交互成本可控，再接 `ARCHIVE` / `EXTRACT` + +- [ ] **Step 4: 把旧读取路径从 `StoredFile.blob` 迁到 `primaryEntity`** + - 覆盖下载、ZIP、分享详情、回收站、媒体元数据读取等旧路径 + - 保留兼容 fallback，直到历史数据回填验证完成 + - 明确哪些 API 已完全不依赖 `blob` + +- [ ] **Step 5: 更新相关测试** + - 后端补读取路径切换和任务入口相关测试 + - 前端补移动端搜索和任务面板交互测试 + +- [ ] **Step 6: 验证** + - `cd backend && mvn test` + - `cd front && npm run test` + - `cd front && npm run lint` + - `cd front && npm run build` + +**Exit criteria:** + +- 桌面与移动端都能搜索 +- 桌面端能直接发起 archive/extract +- 移动端至少能消费任务能力 +- 旧读取链路完成 `primaryEntity` 主读切换 + +--- + +## 6. Stage 3: Thumbnail And Rich Media Pipeline + +**Goal:** 把后台任务骨架扩展成真正可感知的媒体处理系统，这是当前项目相对 Cloudreve 最有价值的缺口。 + +**Files likely involved:** + +- `backend/src/main/java/com/yoyuzh/files/tasks/*` +- `backend/src/main/java/com/yoyuzh/files/storage/*` +- `backend/src/main/java/com/yoyuzh/files/core/*` +- `backend/src/main/java/com/yoyuzh/files/policy/*` +- `backend/src/main/java/com/yoyuzh/files/**/FileMetadata*.java` +- `front/src/pages/Files.tsx` +- `front/src/mobile-pages/MobileFiles.tsx` +- `front/src/lib/types.ts` +- `front/src/components/**/*` +- `backend/src/test/java/com/yoyuzh/files/**` + +- [ ] **Step 1: 增加 `THUMBNAIL` 任务类型与派生实体写入** + - 使用 `FileEntity` 挂接缩略图实体，不再把缩略图视作普通文件 + - 失败时写入可辨识 metadata，避免前端无限重试 + +- [ ] **Step 2: 图片缩略图生成** + - 为常见图片格式生成小尺寸预览 + - 支持本地存储和当前 S3 兼容策略 + +- [ ] **Step 3: 视频 poster 和时长提取** + - 至少落地视频时长 + - poster 可以先做单帧封面，不要求完整 HLS + +- [ ] **Step 4: 前端列表和详情接入缩略图** + - 图片和视频列表显示缩略图 + - 详情侧栏显示时长、尺寸、编码等已有 metadata + - 缩略图不可用时回退到文件图标 + +- [ ] **Step 5: 能力与策略对齐** + - 明确哪些策略支持原生缩略图、哪些需要代理生成 + - 管理台可查看缩略图相关 capability 和任务状态 + +- [ ] **Step 6: 验证** + - `cd backend && mvn test` + - `cd front && npm run test` + - `cd front && npm run lint` + - `cd front && npm run build` + +**Exit criteria:** + +- 上传图片后可自动生成缩略图 +- 上传视频后可得到最小媒体信息和 poster/时长 +- 文件列表和详情可真实消费这些资源 + +--- + +## 7. Stage 4: Metadata, Labels, And Search Expansion + +**Goal:** 让 `FileMetadata` 不再只是骨架，真正承担标签、预览状态和搜索过滤。 + +**Files likely involved:** + +- `backend/src/main/java/com/yoyuzh/files/search/*` +- `backend/src/main/java/com/yoyuzh/files/core/*` +- `backend/src/main/java/com/yoyuzh/files/**/FileMetadata*.java` +- `backend/src/test/java/com/yoyuzh/files/search/*` +- `front/src/lib/file-search.ts` +- `front/src/pages/Files.tsx` +- `front/src/mobile-pages/MobileFiles.tsx` + +- [ ] **Step 1: 明确 metadata key 规范** + - `media:*` + - `thumb:*` + - `tag:*` + - `sys:*` + +- [ ] **Step 2: 扩展 v2 搜索过滤能力** + - 标签过滤 + - 媒体类型过滤 + - 缩略图/预览状态过滤 + +- [ ] **Step 3: 前端增加高级搜索入口** + - 桌面端先做 + - 移动端至少支持基础筛选 + +- [ ] **Step 4: 为后续 `shared_with_me` / `trash` 风格视图保留统一查询边界** + - 当前不必完全复制 Cloudreve File URI + - 但内部查询层应避免继续散落在多套 service 方法里 + +- [ ] **Step 5: 验证** + - `cd backend && mvn test` + - `cd front && npm run test` + - `cd front && npm run lint` + +**Exit criteria:** + +- `FileMetadata` 真正进入用户能力层 +- 搜索不再只停留在文件名和固定字段 + +--- + +## 8. Stage 5: WebDAV Minimum Viable Support + +**Goal:** 提供最小可用的 WebDAV 能力，优先服务系统文件管理器挂载和简单同步场景。 + +**Files likely involved:** + +- `backend/src/main/java/com/yoyuzh/config/*` +- `backend/src/main/java/com/yoyuzh/files/core/*` +- `backend/src/main/java/com/yoyuzh/files/upload/*` +- `backend/src/main/java/com/yoyuzh/auth/*` +- `backend/src/test/java/com/yoyuzh/**` +- `docs/api-reference.md` +- `docs/architecture.md` + +- [ ] **Step 1: 设计 WebDAV 路径边界** + - 首阶段只暴露当前登录用户自己的网盘根目录 + - 不做共享目录挂载 + +- [ ] **Step 2: 实现最小方法集** + - `PROPFIND` + - `GET` + - `PUT` + - `DELETE` + - `MKCOL` + - `MOVE` + +- [ ] **Step 3: 复用现有文件服务和上传会话** + - WebDAV `PUT` 不绕过容量检查、权限检查和存储策略 + - 避免做一套独立写入链路 + +- [ ] **Step 4: 明确认证方式** + - 首阶段优先 Basic + token/应用密码式接入，避免直接复用浏览器 JWT 语义 + - 认证模型必须先文档化，再写实现 + +- [ ] **Step 5: 验证** + - `cd backend && mvn test` + - 手动验证 Windows 或 macOS WebDAV 客户端的列目录、上传小文件、下载、创建目录、删除 + +**Exit criteria:** + +- 常见 WebDAV 客户端能完成最小读写 +- 不引入绕过现有业务规则的旁路实现 + +--- + +## 9. Deferred Backlog + +这些项目保留到 Stage 4 之后重新评估，不在当前一轮升级中直接实现： + +- [ ] **Remote Download** + - 目标：类似 Cloudreve 的离线下载器 + - 前置：后台任务、存储策略、文件实体、容量模型稳定 + +- [ ] **OIDC / OAuth Scope** + - 目标：第三方客户端和更细粒度授权 + - 前置：WebDAV 或开放客户端需求真实存在[text](app://-/index.html?hostId%3Dlocal) + +- [ ] **Desktop Sync Protocol** + - 目标：桌面同步客户端 + - 前置：文件事件、Redis 跨实例广播、冲突策略、WebDAV 或专有同步协议边界明确 + +- [ ] **WOPI / Office Online** + - 目标：在线 Office 协作 + - 前置：权限、锁、预览、第三方接入边界成熟 + +## 10. Recommended Execution Order + +1. Stage 1: 先接 Redis 基础设施 +2. Stage 2: 再补现有闭环断点 +3. Stage 3: 再做缩略图和 richer media +4. Stage 4: 让 metadata/search 真正变成平台能力 +5. Stage 5: 最后接 WebDAV +5. Deferred backlog: 仅在真实需求出现后再启动 + +## 11. Documentation Follow-Up + +每完成一个阶段，都必须同步更新： + +- `memory.md` +- `docs/architecture.md` +- `docs/api-reference.md` + +更新内容至少包括： + +- 新增能力边界 +- 已废弃或迁移的旧路径 +- 验证命令与已知限制 +- 任何新的部署或运行时前置条件 +## 2026-04-10 Stage 1 Step 9 Landing Note + +- 已落地 `FileEventCrossInstancePublisher` + Redis pub/sub listener/publisher：本实例继续维护本地 `SseEmitter` 集合，提交后先做本地广播，再向 `app.redis.namespaces.file-events` 对应 topic 发布事件。 +- 远端实例收到消息后只做本地 SSE 投递，不重复写 `FileEvent` 表；同实例消息按 `instanceId` 忽略，避免本机回环重复推送。 +- Redis 关闭时自动回退为原有单实例本地广播语义。 +## 2026-04-10 Stage 1 Step 10 Landing Note + +- 宸插皢 `AdminService.listStoragePolicies()` 鎺ュ叆 `admin:storage-policies` Spring Cache锛屽苟鍦?`createStoragePolicy/updateStoragePolicy/updateStoragePolicyStatus` 涓夋潯绠＄悊鍐欒矾寰勪笂鍋?all-entries eviction锛岀‘淇濆悗鍙板瓨鍌ㄧ瓥鐣ュ垪琛ㄥ懡涓紦瀛樺悗浠嶈兘鍦ㄧ畝鍗曞啓鎿嶄綔鍚庣珛鍗虫仮澶嶄负鏂版暟鎹€? +- 宸插皢 `AndroidReleaseService.getLatestRelease()` 鎺ュ叆 `android:release` Spring Cache锛屽綋鍓嶉噰鐢?TTL 鍨嬪け鏁堢瓥鐣ワ紝鍥犱负 release metadata 鏇存柊鏉ヨ嚜瀵硅薄瀛樺偍澶栭儴鍙戝竷鑴氭湰锛屼粨搴撳唴娌℃湁鍚屾簮鍐欏叆璺緞銆? +- `admin summary` 缁忚瘎浼板悗鏆備笉鎺ュ叆 Spring Cache锛屽洜涓哄叾鍚屾椂鍖呭惈 request count銆乨aily active users銆乭ourly timeline 绛夐珮棰戠粺璁★紝鍋氭樉寮忓け鏁堝緢闅句繚璇佽涔夊共鍑€锛屽洜姝ゅ湪杩欎竴姝ユ槑纭帓闄ゃ€? +## 2026-04-10 Stage 1 Step 10 Clarification + +- `AdminService.listStoragePolicies()` now uses Spring Cache `admin:storage-policies`. +- Successful storage policy create, update, and status-change writes evict that cache explicitly. +- `AndroidReleaseService.getLatestRelease()` now uses Spring Cache `android:release`. +- Android release metadata refresh is TTL-based because `android/releases/latest.json` is updated by the external release publish script. +- `admin summary` was evaluated and intentionally left uncached because it includes high-churn metrics without a clean explicit invalidation boundary. +## 2026-04-10 Stage 1 Step 11 Clarification + +- `DogeCloudS3SessionProvider` remains an in-process runtime cache and is not moved into Redis. +- The cached value is a live `S3FileRuntimeSession` containing `S3Client` and `S3Presigner`, so cross-instance Redis reuse would add serialization and lifecycle complexity without clear payoff. +- Current semantics remain: each backend instance refreshes its own temporary session only when the cached credentials enter the one-minute refresh window. +- This means multi-instance deployments may fetch duplicate temporary credentials, but the current cost was judged acceptable relative to the extra complexity of a Redis-backed shared credential cache. +- Tests now explicitly cover cache reuse plus refresh-time and close-time resource cleanup in `DogeCloudS3SessionProviderTest`. +## 2026-04-10 Stage 1 Step 12 Clarification + +- Stage 1 validation is complete for the current local environment. +- Full backend verification passed with `cd backend && mvn test`, for a total of 294 passing tests. +- A no-Redis boot-path check also passed under the `dev` profile when the required `APP_JWT_SECRET` environment variable was supplied and `APP_REDIS_ENABLED=false`. +- The local boot verification was captured by starting the backend on port `18081`, confirming that Tomcat started and the application reached the `Started PortalBackendApplication` log line. +- Two earlier local startup failures were confirmed as environment issues rather than Redis regressions: one missing `APP_JWT_SECRET`, and one unrelated port `8080` conflict caused by another local Java process. +- Remaining validation still requires external environment support: a real Redis instance for cache/pubsub/broker/session end-to-end checks, and at least two backend instances for cross-instance event/session propagation checks. + +## 2026-04-10 Stage 1 Step 12 Manual Redis Validation Addendum + +- Stage 1 manual validation was continued in a real local Redis plus dual-backend setup (`dev` profile, ports `18081` and `18082`) after the initial closeout note. +- The backend suite is now green at 301 passing tests after fixing four real Redis/manual-integration regressions discovered during that validation. +- Fix 1: `RedisFileEventPubSubPublisher` and `RedisFileEventPubSubListener` now mark the intended constructor for Spring injection, which unblocked Redis-enabled startup. +- Fix 2: `AuthTokenInvalidationService` now stores and compares access-token revocation cutoffs in epoch seconds, with compatibility handling for earlier millisecond values. +- Fix 3: Redis-backed file list cache now uses the application `ObjectMapper` for Java time serialization and converts generic cache payload maps back into `CachedFileListPage` on cache reads. +- Fix 4: `portal_file.storage_name` is now populated for both directory creation and normal file upload metadata writes, which unblocked real upload/manual event flows against the current schema. +- Manual verification that succeeded in the real Redis plus two-instance setup: +- Re-login invalidates the previous access token and refresh token across instances, while the newest token remains valid. +- Online transfer sessions remain discoverable from the second instance even after the first instance is stopped, which confirms shared runtime state rather than same-process false positives. +- Uploading `image/png` on instance A emits `CREATED` SSE on instance B and auto-creates a queued `MEDIA_META` task visible from instance B. +- Directory list behavior was rechecked through real APIs: repeated `GET /api/files/list` remained stable after the cache fixes, and a subsequent directory mutation was immediately reflected by a fresh list response. +- One environment observation remains open: direct `redis-cli --scan` inspection did not surface the expected Redis keys during local probing, even though cross-instance runtime behavior proved that Redis-backed sharing was active. Treat the runtime behavior checks as the stronger validation result for now. + +## 2026-04-11 Admin Next-Phase Backend Landing Note + +- The next backend-phase admin batch is now landed. +- Implemented admin operational APIs: +- `GET /api/admin/file-blobs` +- `GET /api/admin/shares` +- `DELETE /api/admin/shares/{shareId}` +- `GET /api/admin/tasks` +- `GET /api/admin/tasks/{taskId}` +- The blob admin endpoint is intentionally `FileEntity`-centric and adds operator-facing anomaly signals: `blobMissing`, `orphanRisk`, and `referenceMismatch`. +- The task admin endpoint adds backend-owned parsing for `failureCategory`, `retryScheduled`, `workerOwner`, and `leaseState` so the frontend does not need to infer them from raw JSON. +- Integration and service coverage were expanded in `AdminControllerIntegrationTest` and `AdminServiceTest`, and the storage-policy cache test was kept aligned with the current constructor/dependency graph. +- Verification passed with targeted admin tests and full backend regression: +- `cd backend && mvn -Dtest=AdminControllerIntegrationTest,AdminServiceTest,AdminServiceStoragePolicyCacheTest test` +- `cd backend && mvn test` +- Full backend result after this landing note: 304 tests passed. diff --git a/memory.md b/memory.md index bd8bf0d..12b2808 100644 --- a/memory.md +++ b/memory.md @@ -186,3 +186,108 @@ - 2026-04-09 存储策略管理后端继续收口：管理员接口已从只读 `GET /api/admin/storage-policies` 扩展到 `POST /api/admin/storage-policies`、`PUT /api/admin/storage-policies/{policyId}`、`PATCH /api/admin/storage-policies/{policyId}/status` 和 `POST /api/admin/storage-policies/migrations`。当前支持新增、编辑、启停非默认策略，并可创建 `STORAGE_POLICY_MIGRATION` 后台任务；默认策略不能停用，仍不支持删除策略或切换默认策略。 - 2026-04-09 存储策略与上传路径后端继续推进：`STORAGE_POLICY_MIGRATION` 现已从 skeleton 升级为“当前活动存储后端内的真实迁移”。worker 会限制源/目标策略必须同类型，读取旧 `FileBlob` 对象字节，写入新的 `policies/{targetPolicyId}/blobs/...` object key，同步更新 `FileBlob.objectKey` 与 `FileEntity.VERSION(objectKey, storagePolicyId)`，并在事务提交后异步清理旧对象；若处理中失败，会删除本轮新写对象并依赖事务回滚元数据。与此同时，v2 upload session 现在会按默认策略能力决策 `uploadMode=PROXY|DIRECT_SINGLE|DIRECT_MULTIPART`：`directUpload=false` 时走 `POST /api/v2/files/upload-sessions/{sessionId}/content` 代理上传，`directUpload=true && multipartUpload=false` 时走 `GET /api/v2/files/upload-sessions/{sessionId}/prepare` 单请求直传，`multipartUpload=true` 时继续走现有分片 prepare/record/complete 链路；会话响应还会附带 `strategy`，把当前模式下的后续后端入口模板显式返回给前端；旧 `/api/files/upload/initiate` 也会尊重默认策略的 `directUpload/maxObjectSize`。 - 2026-04-09 前端 files 上传链路已切到 v2 upload session：桌面端 `FilesPage`、移动端 `MobileFilesPage` 和 `saveFileToNetdisk()` 现在统一通过 `front/src/lib/upload-session.ts` 走 `create/get/cancel/prepare/content/part-prepare/part-record/complete` 全套 helper，并按后端返回的 `uploadMode + strategy` 自动选择 `PROXY / DIRECT_SINGLE / DIRECT_MULTIPART`。旧 `/api/files/upload/**` 当前仍保留给头像等非 files 子系统入口使用。 +- 2026-04-10 存储策略与上传路径后端进入正式迁移，并完成前端视觉系统全面升级： + - 后端：`STORAGE_POLICY_MIGRATION` 任务逻辑完整化，支持同类型后端间的数据物理迁移与元数据同步；v2 upload session 现已按策略能力矩阵分发 `PROXY / DIRECT_SINGLE / DIRECT_MULTIPART` 策略。 + - 前端视觉：全站 UI 已重构为“Stitch”玻璃拟态 (Glassmorphism) 风格。引入全局 `bg-aurora` 背景、`.glass-panel` 通用样式类、`ThemeProvider` 与 `ThemeToggle` 亮暗色切换。 + - 前端模块：网盘、快传、分享、任务、回收站、移动端布局、管理台 Dashboard、用户、文件、存储策略等所有核心视图均已完成视觉重构，在保持原有数据绑定与逻辑闭环的前提下，实现了极高质感的 UI 表现。 + - 前端技术栈：由于 `front/` 根目录不直接由 UI 框架管理，通过 `src/components/` 及其对应 hooks/lib 实现了一套自定义的主题与玻璃态组件库，并解决了 overhaul 过程中引入的所有 TypeScript / Lint 缺失引用问题。 +- 2026-04-10 Cloudreve gap 后端升级计划已完成 Stage 1 第一批： + - 新增 Spring Cache 与 Spring Data Redis 依赖，`application.yml` / `application-dev.yml` 增加 `spring.data.redis.*` 与默认关闭的 `app.redis.*` 配置骨架；`spring.data.redis.repositories.enabled=false`，当前不启用 Redis repository。 + - 新增 `AppRedisProperties`、`RedisConfiguration`、`RedisCacheNames`，把 Redis 使用边界拆成 `cache/auth/transfer-sessions/upload-state/locks/file-events/broker` 命名空间；Redis 关闭时回退到 `NoOpCacheManager`，不强依赖本地或 dev 环境外部 Redis。 + - 新增 `AuthTokenInvalidationService`：Redis 启用时按 `userId + clientType` 写入 access token 的失效时间标记，并把被撤销 refresh token 的 hash 以剩余有效期 TTL 写入 Redis 黑名单；Redis 关闭时自动使用 no-op 实现。 + - `AuthService` 的同端重登与改密、`AdminService` 的封禁/改密/重置密码、`RefreshTokenService` 的轮换/批量撤销/过期拒绝，现已统一接到这套 Redis 登录态失效层。 + - `JwtAuthenticationFilter` 现在会在原有 JWT + `sid` 校验前先检查 Redis access token 失效标记；快传 session、热目录缓存、分布式锁、文件事件跨实例广播和轻量 broker 仍留在后续 Stage 1 小步。 +## 2026-04-10 Stage 1 Batch 2 + +- `/api/files/list` 现已接入可选 Redis 热目录分页缓存，缓存 key 固定包含 `userId + path + page + size + sort context + directory version`，并在创建、删除、移动、复制、重命名、恢复、上传完成和导入后按目录版本精准失效。 +- 第一批分布式锁已落在回收站恢复路径，`FileService.restoreFromRecycleBin(...)` 通过 Redis `locks` 命名空间做带 TTL 和 owner token 的互斥，避免同一条目被并发恢复。 +- 上传会话短状态现已进入 Redis `upload-state` 命名空间，`UploadSessionService` 会在创建、上传中、完成、取消、失败、过期时刷新运行态；`GET /api/v2/files/upload-sessions/{sessionId}` 响应新增 `runtime` 字段，前端可直接读取 phase、uploadedBytes、uploadedPartCount、progressPercent、lastUpdatedAt、expiresAt。 +- 这一批后端升级已通过 `cd backend && mvn test` 全量验证，结果为 277 tests passed。 + +## 2026-04-10 Stage 1 Batch 3 + +- Stage 1 Step 7 已落地首批轻量 broker：新增 `LightweightBrokerService` 抽象，Redis 启用时走 Redis list，Redis 关闭时回退到内存队列，继续支持本地单实例开发和测试。 +- 当前 broker 的首个真实用例是媒体任务自动触发：`FileService.saveFileMetadata(...)` 会在媒体文件元数据落库并提交事务后，通过 `MediaMetadataTaskBrokerPublisher` 发布 `media-metadata-trigger`。 +- `MediaMetadataTaskBrokerConsumer` 会批量 drain 这类消息，并调用 `BackgroundTaskService.createQueuedAutoMediaMetadataTask(...)` 创建 `MEDIA_META` 后台任务；创建前会按 `correlationId` 去重，并重新校验文件仍存在、未删除且仍是媒体文件。 +- 这批 broker 明确不是高可靠消息系统，也不替代现有数据库 `BackgroundTask` worker；文件事件跨实例广播仍留给 Stage 1 Step 9 的 Redis pub/sub。 +- 本批次新增/更新测试后，`cd backend && mvn test` 已通过，结果为 281 tests passed。 + +## 2026-04-10 Stage 1 Batch 4 + +- Stage 1 Step 8 已完成：在线快传 `TransferSessionStore` 不再只依赖进程内 `ConcurrentHashMap`，Redis 启用时会把 session 快照与 `pickupCode -> sessionId` 映射写入 `transfer-sessions` 命名空间；Redis 关闭时自动回退到内存模式。 +- `TransferSession` 新增内部快照序列化形状，保留 `receiverJoined`、信令队列、cursor 和文件清单等在线运行态；因此 `joinSession` 和 `postSignal` 在修改在线会话后会重新写回 store，避免 Redis 模式下状态只改在临时副本里。 +- `TransferService.nextPickupCode()` 现已复用 store 侧生成逻辑；Redis 启用时会先对 pickup code 做短 TTL 预留，降低多实例并发创建在线快传 session 的碰撞概率。 +- 当前这一步只覆盖在线快传跨实例共享；离线快传仍继续走数据库 `OfflineTransferSessionRepository`，文件事件跨实例广播仍留给 Stage 1 Step 9。 +- 本批次补了 `TransferServiceTest` 和 `TransferSessionStoreTest`，并已通过 `mvn -Dtest=TransferControllerIntegrationTest,TransferServiceTest,TransferSessionStoreTest test` 与 `cd backend && mvn test`；全量结果为 284 tests passed。 +## 2026-04-10 Stage 1 Batch 5 + +- Stage 1 Step 9 已完成：文件事件从“仅单实例内存广播”升级为“本地 SSE 广播 + Redis pub/sub 跨实例转发”。本地订阅管理仍留在 `FileEventService` 的内存 `subscriptions`，没有把 `SseEmitter` 或订阅状态存进 Redis。 +- 新增 `FileEventCrossInstancePublisher` 抽象与 Redis/no-op 双实现；Redis 开启时，`RedisFileEventPubSubPublisher` 会把已提交的 `FileEvent` 最小快照发布到 `keyPrefix:file-events:pubsub`，并附带当前实例 `instanceId`。 +- `RedisFileEventPubSubListener` 会订阅同一 topic，忽略本实例回环消息，只把远端事件重建后交给 `FileEventService.broadcastReplicatedEvent(...)` 做本地 SSE 投递，因此不会重复写 `FileEvent` 表。 +- 这批实现明确只解决“多实例下文件事件能到达其它实例上的活跃 SSE 订阅”问题，不提供历史重放、可靠投递或补偿语义；事件持久化事实源仍然是数据库 `portal_file_event`。 +- 验证已覆盖 `FileEventServiceTest`、`RedisFileEventPubSubPublisherTest`、`RedisFileEventPubSubListenerTest`、既有 `FileEventPersistenceIntegrationTest`、`FileEventsV2ControllerIntegrationTest`，并通过 `cd backend && mvn test`；全量结果更新为 288 tests passed。 +## 2026-04-10 Stage 1 Batch 6 + +- Stage 1 Step 10 宸插畬鎴愶細`AdminService.listStoragePolicies()` 鎺ュ叆 `admin:storage-policies` Spring Cache锛屽悗鍙板瓨鍌ㄧ瓥鐣ュ垪琛ㄧ幇鍦ㄤ細鍦?create/update/status 鍐欐搷浣滃悗鍋?all-entries eviction锛汻edis 鍏抽棴鏃朵粛鑷姩鍥為€€鍒板師鏈夐潪缂撳瓨璇昏矾寰勩€? +- `AndroidReleaseService.getLatestRelease()` 鐜板凡鎺ュ叆 `android:release` Spring Cache锛屽綋鍓嶉€氳繃 TTL 鎺у埗鏁版嵁鍒锋柊锛涘洜涓哄畨鍗撳彂甯冨厓鏁版嵁鏄敱浠撳簱澶栫殑瀵硅薄瀛樺偍鍙戝竷鑴氭湰鏇存柊锛屾病鏈夊悓婧愬啓璺緞鍙互鍦ㄥ悗绔唴閮ㄦ樉寮忓け鏁堛€? +- `admin summary` 缁忚瘎浼板悗鏆備笉缂撳瓨锛屽洜涓哄叾鍚屾椂鍖呭惈 request count銆乨aily active users銆乭ourly timeline 绛夐珮棰戠粺璁″€硷紝鍋氭樉寮忓け鏁堜細璁╄涔夊彉寰椾笉绋冲畾銆? +- 杩欐壒琛ヤ簡 `AdminServiceStoragePolicyCacheTest` 鍜?`AndroidReleaseServiceCacheTest` 锛屽苟閫氳繃 `mvn -Dtest=AdminControllerIntegrationTest,AndroidReleaseServiceTest,AndroidReleaseControllerTest,AdminServiceStoragePolicyCacheTest,AndroidReleaseServiceCacheTest test` 涓?`cd backend && mvn test`锛屽叏閲忕粨鏋滄洿鏂颁负 293 tests passed銆? +## 2026-04-10 Stage 1 Batch 6 Clarification + +- Step 10 is complete. +- `AdminService.listStoragePolicies()` now uses Spring Cache `admin:storage-policies`. +- Successful storage policy create, update, and status-change writes evict that cache. +- `AndroidReleaseService.getLatestRelease()` now uses Spring Cache `android:release`. +- Android release metadata refresh is TTL-driven because updates come from the external release publish script writing `android/releases/latest.json`. +- `admin summary` was evaluated and intentionally left uncached because it includes high-churn metrics without a clean explicit invalidation boundary. +- Verification passed with targeted cache/admin/android tests and full `cd backend && mvn test`. +- Full backend result after this batch: 293 tests passed. +## 2026-04-10 Stage 1 Batch 7 Clarification + +- Stage 1 Step 11 is complete with a deliberate non-change: `DogeCloudS3SessionProvider` stays as a per-instance in-memory runtime cache. +- The provider caches a live `S3FileRuntimeSession` (`S3Client` + `S3Presigner`) and refreshes only when the temporary credentials enter the built-in one-minute refresh window. +- Multi-instance duplicate temporary-token fetches were judged acceptable; the repo does not now add Redis-based shared credential caching for DogeCloud temporary S3 sessions. +- `DogeCloudS3SessionProviderTest` now also covers refresh-time cleanup of the previous runtime session and explicit `close()` cleanup. +## 2026-04-10 Stage 1 Batch 8 Clarification + +- Stage 1 Step 12 is complete as a validation closeout batch. +- Local verification passed with full `cd backend && mvn test`, keeping the backend suite green at 294 passing tests. +- Redis-disabled boot compatibility was also re-checked: with `APP_REDIS_ENABLED=false`, `APP_JWT_SECRET` set, and `dev` profile active, the backend booted successfully and reached `Started PortalBackendApplication` on port `18081`. +- This confirms the new Redis-backed capabilities still preserve the no-Redis local-development path instead of making Redis a hard startup dependency. +- What remains unverified locally is environment-bound rather than code-bound: real Redis end-to-end behavior and multi-instance propagation for pub/sub, lightweight broker consumption, and Redis-backed runtime/session sharing. + +## 2026-04-10 Stage 1 Batch 9 Manual Redis Validation + +- Stage 1 manual Redis validation was continued with a real local Redis service plus two backend instances on `18081` and `18082`. +- Four real regressions were found and fixed during that validation: +- `RedisFileEventPubSubPublisher` and `RedisFileEventPubSubListener` needed explicit constructor selection for Spring bean creation in Redis-enabled startup. +- `AuthTokenInvalidationService` was writing revocation cutoffs in milliseconds while JWT `iat` comparison effectively worked at second precision, causing fresh tokens to be treated as revoked; it now stores epoch seconds and tolerates old millisecond Redis values. +- Redis file list cache needed two runtime fixes: cache serialization must use the application `ObjectMapper` so `LocalDateTime` can be written, and cache reads must tolerate generic map payloads returned by Redis cache deserialization. +- `portal_file.storage_name` was missing in both `mkdir` and normal file upload metadata writes against the current schema, so both paths now persist a non-null legacy storage name. +- Manual multi-instance verification that actually passed: +- re-login invalidates the old access token and old refresh token while keeping the latest token usable; +- online transfer lookup still works from instance B after instance A is stopped, proving shared runtime state; +- uploading `image/png` on instance A delivers a `CREATED` SSE event to instance B and auto-creates one queued `MEDIA_META` task visible from instance B. +- Backend test count is now 301 passing tests after adding coverage for the new Redis/manual-integration regressions. +- A remaining environment note: direct `redis-cli` key scans did not show the expected Redis keys during local probing even though the cross-instance runtime checks proved Redis-backed sharing was active, so runtime behavior is currently the stronger evidence than raw key inspection. + +## Debugging Discipline + +- Use short bounded probes first when validating network, dependency, or startup issues. Prefer commands such as `curl --max-time`, `mvn -q`, `mvn dependency:get`, `apt-get update`, and similar narrow checks before launching long-running downloads or full test runs. +- Do not wait indefinitely on a stalled download or progress indicator. If a command appears stuck, stop and re-check DNS, proxy inheritance, mirror reachability, and direct-vs-proxy routing before retrying. +- For WSL debugging, verify the proxy path and the direct path separately, then choose the shortest working route. Do not assume a mirror problem until the network path has been isolated. +- Use domestic mirrors as a delivery optimization, not as a substitute for diagnosis. First determine whether the failure is caused by DNS, proxy configuration, upstream availability, or the mirror itself. + +## 2026-04-11 Admin Backend Surface Addendum + +- The next backend phase from `2026-04-10-cloudreve-gap-next-phase-upgrade.md` is now underway on the admin surface. +- `AdminController` and `AdminService` now expose three new admin data areas: +- `GET /api/admin/file-blobs`: entity-centric blob inspection across `FileEntity`, `StoredFileEntity`, and `FileBlob`, including `blobMissing`, `orphanRisk`, and `referenceMismatch` signals. +- `GET /api/admin/shares` and `DELETE /api/admin/shares/{shareId}`: admin-side share listing and forced cleanup for `FileShareLink`. +- `GET /api/admin/tasks` and `GET /api/admin/tasks/{taskId}`: admin-side background task inspection with parsed `failureCategory`, `retryScheduled`, `workerOwner`, and derived `leaseState`. +- The blob admin list is intentionally based on `FileEntity` instead of `StoredFile` so storage-policy migration and future multi-entity object lifecycles can be inspected without relying on the legacy `StoredFile.blob` read path. +- Old public/user read flows still intentionally depend on `StoredFile.blob`; this batch does not yet switch download/share/recycle/zip reads to `primaryEntity`. +- Verification for this batch passed with: +- `cd backend && mvn -Dtest=AdminControllerIntegrationTest,AdminServiceTest,AdminServiceStoragePolicyCacheTest test` +- `cd backend && mvn test` +- Full backend result after this addendum: 304 tests passed.