From f582e600aaff280e8d96391f261288b332dcc1b1 Mon Sep 17 00:00:00 2001 From: yoyuzh Date: Wed, 8 Apr 2026 15:27:39 +0800 Subject: [PATCH] feat(files): expire stale upload sessions --- .../yoyuzh/files/UploadSessionService.java | 36 ++++++++++++++++++- .../files/UploadSessionServiceTest.java | 25 +++++++++++++ docs/api-reference.md | 1 + docs/architecture.md | 1 + memory.md | 1 + 5 files changed, 63 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/yoyuzh/files/UploadSessionService.java b/backend/src/main/java/com/yoyuzh/files/UploadSessionService.java index 9d93fce..3e1d4b3 100644 --- a/backend/src/main/java/com/yoyuzh/files/UploadSessionService.java +++ b/backend/src/main/java/com/yoyuzh/files/UploadSessionService.java @@ -6,7 +6,9 @@ import com.yoyuzh.auth.User; import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.ErrorCode; import com.yoyuzh.config.FileStorageProperties; +import com.yoyuzh.files.storage.FileContentStorage; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; @@ -23,12 +25,18 @@ public class UploadSessionService { private static final long DEFAULT_CHUNK_SIZE = 8L * 1024 * 1024; private static final long SESSION_TTL_HOURS = 24; + private static final List EXPIRABLE_STATUSES = List.of( + UploadSessionStatus.CREATED, + UploadSessionStatus.UPLOADING, + UploadSessionStatus.COMPLETING + ); private static final TypeReference> UPLOADED_PARTS_TYPE = new TypeReference<>() { }; private final UploadSessionRepository uploadSessionRepository; private final StoredFileRepository storedFileRepository; private final FileService fileService; + private final FileContentStorage fileContentStorage; private final ObjectMapper objectMapper = new ObjectMapper(); private final long maxFileSize; private final Clock clock; @@ -37,18 +45,21 @@ public class UploadSessionService { public UploadSessionService(UploadSessionRepository uploadSessionRepository, StoredFileRepository storedFileRepository, FileService fileService, + FileContentStorage fileContentStorage, FileStorageProperties properties) { - this(uploadSessionRepository, storedFileRepository, fileService, properties, Clock.systemUTC()); + this(uploadSessionRepository, storedFileRepository, fileService, fileContentStorage, properties, Clock.systemUTC()); } UploadSessionService(UploadSessionRepository uploadSessionRepository, StoredFileRepository storedFileRepository, FileService fileService, + FileContentStorage fileContentStorage, FileStorageProperties properties, Clock clock) { this.uploadSessionRepository = uploadSessionRepository; this.storedFileRepository = storedFileRepository; this.fileService = fileService; + this.fileContentStorage = fileContentStorage; this.maxFileSize = properties.getMaxFileSize(); this.clock = clock; } @@ -166,6 +177,29 @@ public class UploadSessionService { } } + @Scheduled(fixedDelay = 60 * 60 * 1000L) + @Transactional + public int pruneExpiredSessions() { + LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); + List expiredSessions = uploadSessionRepository.findByStatusInAndExpiresAtBefore( + EXPIRABLE_STATUSES, + now + ); + for (UploadSession session : expiredSessions) { + try { + fileContentStorage.deleteBlob(session.getObjectKey()); + } catch (RuntimeException ignored) { + // Expiration is authoritative in the database even if remote object cleanup fails. + } + session.setStatus(UploadSessionStatus.EXPIRED); + session.setUpdatedAt(now); + } + if (!expiredSessions.isEmpty()) { + uploadSessionRepository.saveAll(expiredSessions); + } + return expiredSessions.size(); + } + private void validateTarget(User user, String normalizedPath, String filename, long size) { long effectiveMaxUploadSize = Math.min(maxFileSize, user.getMaxUploadSizeBytes()); if (size > effectiveMaxUploadSize) { diff --git a/backend/src/test/java/com/yoyuzh/files/UploadSessionServiceTest.java b/backend/src/test/java/com/yoyuzh/files/UploadSessionServiceTest.java index 75b5bcf..e35567b 100644 --- a/backend/src/test/java/com/yoyuzh/files/UploadSessionServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/files/UploadSessionServiceTest.java @@ -3,6 +3,7 @@ package com.yoyuzh.files; import com.yoyuzh.auth.User; import com.yoyuzh.common.BusinessException; import com.yoyuzh.config.FileStorageProperties; +import com.yoyuzh.files.storage.FileContentStorage; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -14,11 +15,13 @@ import java.time.Clock; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.List; import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.mockito.Mockito.verify; @@ -32,6 +35,8 @@ class UploadSessionServiceTest { private StoredFileRepository storedFileRepository; @Mock private FileService fileService; + @Mock + private FileContentStorage fileContentStorage; private UploadSessionService uploadSessionService; @@ -43,6 +48,7 @@ class UploadSessionServiceTest { uploadSessionRepository, storedFileRepository, fileService, + fileContentStorage, properties, Clock.fixed(Instant.parse("2026-04-08T06:00:00Z"), ZoneOffset.UTC) ); @@ -179,6 +185,25 @@ class UploadSessionServiceTest { )).isInstanceOf(BusinessException.class); } + @Test + void shouldExpireUnfinishedSessionsAndDeleteTemporaryBlobs() { + User user = createUser(7L); + UploadSession session = createSession(user); + session.setStatus(UploadSessionStatus.UPLOADING); + session.setObjectKey("blobs/expired-session"); + session.setExpiresAt(LocalDateTime.of(2026, 4, 8, 5, 0)); + when(uploadSessionRepository.findByStatusInAndExpiresAtBefore(anyList(), eq(LocalDateTime.of(2026, 4, 8, 6, 0)))) + .thenReturn(List.of(session)); + + int expiredCount = uploadSessionService.pruneExpiredSessions(); + + assertThat(expiredCount).isEqualTo(1); + assertThat(session.getStatus()).isEqualTo(UploadSessionStatus.EXPIRED); + assertThat(session.getUpdatedAt()).isEqualTo(LocalDateTime.of(2026, 4, 8, 6, 0)); + verify(fileContentStorage).deleteBlob("blobs/expired-session"); + verify(uploadSessionRepository).saveAll(List.of(session)); + } + private User createUser(Long id) { User user = new User(); user.setId(id); diff --git a/docs/api-reference.md b/docs/api-reference.md index 9ba390f..c9fc685 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -444,3 +444,4 @@ - 2026-04-08 阶段 3 第一小步 API 补充：新增受保护的 v2 上传会话骨架接口，`POST /api/v2/files/upload-sessions` 创建会话，`GET /api/v2/files/upload-sessions/{sessionId}` 查询当前用户自己的会话，`DELETE /api/v2/files/upload-sessions/{sessionId}` 取消会话。当前响应只返回 `sessionId`、`objectKey`、路径、文件名、状态、分片大小、分片数量和时间字段；实际文件内容仍走旧上传链路，尚未开放 v2 分片上传/完成接口。 - 2026-04-08 阶段 3 第二小步 API 补充：新增 `POST /api/v2/files/upload-sessions/{sessionId}/complete`，用于把当前用户自己的上传会话提交完成。该接口当前不接收请求体，会复用会话里的 `objectKey/path/filename/contentType/size` 调用旧上传完成落库链路，成功后返回 `COMPLETED` 状态的 v2 会话响应；分片内容上传端点仍未开放。 - 2026-04-08 阶段 3 第三小步 API 补充：新增 `PUT /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}`，请求体为 `{ "etag": "...", "size": 8388608 }`，用于记录当前用户上传会话的 part 元数据并返回 v2 会话响应。该接口会校验 part 范围和会话状态，当前只更新 `uploadedPartsJson`，不接收或合并真实文件分片内容。 +- 2026-04-08 阶段 3 第四小步 API 补充：本小步没有新增对外 API。后端新增上传会话过期清理任务，只处理未完成且已过期的会话，并把它们标记为 `EXPIRED`；已完成会话和旧 `/api/files/**` 上传接口响应不变。 diff --git a/docs/architecture.md b/docs/architecture.md index 24846f8..5598b94 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -448,3 +448,4 @@ Android 壳补充说明： - 2026-04-08 阶段 3 第一小步补充：后端新增上传会话二期最小骨架。`UploadSession` 记录用户、目标路径、文件名、对象键、分片大小、分片数量、状态、过期时间和已上传分片占位 JSON；`/api/v2/files/upload-sessions` 目前只提供创建、查询、取消会话，不承接实际分片内容上传，也不替换旧 `/api/files/upload/**` 生产链路。 - 2026-04-08 阶段 3 第二小步补充：上传会话新增完成状态机。`UploadSessionService.completeOwnedSession()` 会复用旧 `FileService.completeUpload()` 完成对象确认、目录补齐、配额/冲突校验和 `FileBlob + StoredFile + FileEntity.VERSION` 双写落库，然后把会话标记为 `COMPLETED`；失败时标记 `FAILED`，过期时标记 `EXPIRED`。当前仍没有独立 v2 分片内容写入端点。 - 2026-04-08 阶段 3 第三小步补充：上传会话新增 part 状态记录。`UploadSessionService.recordUploadedPart()` 会校验会话归属、状态、过期时间和 part 范围，把 `etag/size/uploadedAt` 写入 `uploadedPartsJson`，并将新会话推进到 `UPLOADING`。当前实现是会话状态跟踪，不是跨存储驱动的分片内容写入/合并实现。 +- 2026-04-08 阶段 3 第四小步补充：上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话，尝试删除 `objectKey` 对应的临时 blob，然后标记为 `EXPIRED`。已完成文件不参与清理，避免误删已经落库的生产对象。 diff --git a/memory.md b/memory.md index 4465294..e728326 100644 --- a/memory.md +++ b/memory.md @@ -162,3 +162,4 @@ - 2026-04-08 阶段 3 第一小步：新增后端上传会话二期最小骨架，包含 `UploadSession`、`UploadSessionStatus`、`UploadSessionRepository`、`UploadSessionService`，以及受保护的 `/api/v2/files/upload-sessions` 创建、查询、取消接口；旧 `/api/files/upload/**` 上传链路暂不切换，前端上传队列暂不改动。 - 2026-04-08 阶段 3 第二小步：新增 `POST /api/v2/files/upload-sessions/{sessionId}/complete`，v2 上传会话可从 `CREATED` 进入 `COMPLETING` 并复用旧 `FileService.completeUpload()` 完成 `FileBlob + StoredFile + FileEntity.VERSION` 落库，成功后标记 `COMPLETED`；取消、失败、过期会话不能完成。实际分片内容上传和前端上传队列仍未切换。 - 2026-04-08 阶段 3 第三小步：新增 `PUT /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}`，用于记录当前用户上传会话的 part 元数据到 `uploadedPartsJson`，并把会话状态从 `CREATED` 推进到 `UPLOADING`；该接口只记录 `etag/size` 等状态，不承担真正的对象存储分片内容写入或合并。 +- 2026-04-08 阶段 3 第四小步：`UploadSessionService` 新增定时过期清理，按小时扫描 `CREATED/UPLOADING/COMPLETING` 且已过期的会话，尝试删除对应临时 `blobs/...` 对象，并把会话标记为 `EXPIRED`；`COMPLETED/CANCELLED/FAILED/EXPIRED` 不在本轮清理范围内。