feat(files): expire stale upload sessions
This commit is contained in:
yoyuzh
@@ -6,7 +6,9 @@ import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.ErrorCode;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.StringUtils;
|
||||
@@ -23,12 +25,18 @@ public class UploadSessionService {
|
||||
|
||||
private static final long DEFAULT_CHUNK_SIZE = 8L * 1024 * 1024;
|
||||
private static final long SESSION_TTL_HOURS = 24;
|
||||
private static final List<UploadSessionStatus> EXPIRABLE_STATUSES = List.of(
|
||||
UploadSessionStatus.CREATED,
|
||||
UploadSessionStatus.UPLOADING,
|
||||
UploadSessionStatus.COMPLETING
|
||||
);
|
||||
private static final TypeReference<List<UploadedPart>> UPLOADED_PARTS_TYPE = new TypeReference<>() {
|
||||
};
|
||||
|
||||
private final UploadSessionRepository uploadSessionRepository;
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final FileService fileService;
|
||||
private final FileContentStorage fileContentStorage;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final long maxFileSize;
|
||||
private final Clock clock;
|
||||
@@ -37,18 +45,21 @@ public class UploadSessionService {
|
||||
public UploadSessionService(UploadSessionRepository uploadSessionRepository,
|
||||
StoredFileRepository storedFileRepository,
|
||||
FileService fileService,
|
||||
FileContentStorage fileContentStorage,
|
||||
FileStorageProperties properties) {
|
||||
this(uploadSessionRepository, storedFileRepository, fileService, properties, Clock.systemUTC());
|
||||
this(uploadSessionRepository, storedFileRepository, fileService, fileContentStorage, properties, Clock.systemUTC());
|
||||
}
|
||||
|
||||
UploadSessionService(UploadSessionRepository uploadSessionRepository,
|
||||
StoredFileRepository storedFileRepository,
|
||||
FileService fileService,
|
||||
FileContentStorage fileContentStorage,
|
||||
FileStorageProperties properties,
|
||||
Clock clock) {
|
||||
this.uploadSessionRepository = uploadSessionRepository;
|
||||
this.storedFileRepository = storedFileRepository;
|
||||
this.fileService = fileService;
|
||||
this.fileContentStorage = fileContentStorage;
|
||||
this.maxFileSize = properties.getMaxFileSize();
|
||||
this.clock = clock;
|
||||
}
|
||||
@@ -166,6 +177,29 @@ public class UploadSessionService {
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 60 * 60 * 1000L)
|
||||
@Transactional
|
||||
public int pruneExpiredSessions() {
|
||||
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone());
|
||||
List<UploadSession> expiredSessions = uploadSessionRepository.findByStatusInAndExpiresAtBefore(
|
||||
EXPIRABLE_STATUSES,
|
||||
now
|
||||
);
|
||||
for (UploadSession session : expiredSessions) {
|
||||
try {
|
||||
fileContentStorage.deleteBlob(session.getObjectKey());
|
||||
} catch (RuntimeException ignored) {
|
||||
// Expiration is authoritative in the database even if remote object cleanup fails.
|
||||
}
|
||||
session.setStatus(UploadSessionStatus.EXPIRED);
|
||||
session.setUpdatedAt(now);
|
||||
}
|
||||
if (!expiredSessions.isEmpty()) {
|
||||
uploadSessionRepository.saveAll(expiredSessions);
|
||||
}
|
||||
return expiredSessions.size();
|
||||
}
|
||||
|
||||
private void validateTarget(User user, String normalizedPath, String filename, long size) {
|
||||
long effectiveMaxUploadSize = Math.min(maxFileSize, user.getMaxUploadSizeBytes());
|
||||
if (size > effectiveMaxUploadSize) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.yoyuzh.files;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -14,11 +15,13 @@ import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.mockito.Mockito.verify;
|
||||
@@ -32,6 +35,8 @@ class UploadSessionServiceTest {
|
||||
private StoredFileRepository storedFileRepository;
|
||||
@Mock
|
||||
private FileService fileService;
|
||||
@Mock
|
||||
private FileContentStorage fileContentStorage;
|
||||
|
||||
private UploadSessionService uploadSessionService;
|
||||
|
||||
@@ -43,6 +48,7 @@ class UploadSessionServiceTest {
|
||||
uploadSessionRepository,
|
||||
storedFileRepository,
|
||||
fileService,
|
||||
fileContentStorage,
|
||||
properties,
|
||||
Clock.fixed(Instant.parse("2026-04-08T06:00:00Z"), ZoneOffset.UTC)
|
||||
);
|
||||
@@ -179,6 +185,25 @@ class UploadSessionServiceTest {
|
||||
)).isInstanceOf(BusinessException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldExpireUnfinishedSessionsAndDeleteTemporaryBlobs() {
|
||||
User user = createUser(7L);
|
||||
UploadSession session = createSession(user);
|
||||
session.setStatus(UploadSessionStatus.UPLOADING);
|
||||
session.setObjectKey("blobs/expired-session");
|
||||
session.setExpiresAt(LocalDateTime.of(2026, 4, 8, 5, 0));
|
||||
when(uploadSessionRepository.findByStatusInAndExpiresAtBefore(anyList(), eq(LocalDateTime.of(2026, 4, 8, 6, 0))))
|
||||
.thenReturn(List.of(session));
|
||||
|
||||
int expiredCount = uploadSessionService.pruneExpiredSessions();
|
||||
|
||||
assertThat(expiredCount).isEqualTo(1);
|
||||
assertThat(session.getStatus()).isEqualTo(UploadSessionStatus.EXPIRED);
|
||||
assertThat(session.getUpdatedAt()).isEqualTo(LocalDateTime.of(2026, 4, 8, 6, 0));
|
||||
verify(fileContentStorage).deleteBlob("blobs/expired-session");
|
||||
verify(uploadSessionRepository).saveAll(List.of(session));
|
||||
}
|
||||
|
||||
private User createUser(Long id) {
|
||||
User user = new User();
|
||||
user.setId(id);
|
||||
|
||||
@@ -444,3 +444,4 @@
|
||||
- 2026-04-08 阶段 3 第一小步 API 补充：新增受保护的 v2 上传会话骨架接口，`POST /api/v2/files/upload-sessions` 创建会话，`GET /api/v2/files/upload-sessions/{sessionId}` 查询当前用户自己的会话，`DELETE /api/v2/files/upload-sessions/{sessionId}` 取消会话。当前响应只返回 `sessionId`、`objectKey`、路径、文件名、状态、分片大小、分片数量和时间字段；实际文件内容仍走旧上传链路，尚未开放 v2 分片上传/完成接口。
|
||||
- 2026-04-08 阶段 3 第二小步 API 补充：新增 `POST /api/v2/files/upload-sessions/{sessionId}/complete`，用于把当前用户自己的上传会话提交完成。该接口当前不接收请求体，会复用会话里的 `objectKey/path/filename/contentType/size` 调用旧上传完成落库链路，成功后返回 `COMPLETED` 状态的 v2 会话响应；分片内容上传端点仍未开放。
|
||||
- 2026-04-08 阶段 3 第三小步 API 补充：新增 `PUT /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}`，请求体为 `{ "etag": "...", "size": 8388608 }`，用于记录当前用户上传会话的 part 元数据并返回 v2 会话响应。该接口会校验 part 范围和会话状态，当前只更新 `uploadedPartsJson`，不接收或合并真实文件分片内容。
|
||||
- 2026-04-08 阶段 3 第四小步 API 补充：本小步没有新增对外 API。后端新增上传会话过期清理任务，只处理未完成且已过期的会话，并把它们标记为 `EXPIRED`；已完成会话和旧 `/api/files/**` 上传接口响应不变。
|
||||
|
||||
@@ -448,3 +448,4 @@ Android 壳补充说明：
|
||||
- 2026-04-08 阶段 3 第一小步补充：后端新增上传会话二期最小骨架。`UploadSession` 记录用户、目标路径、文件名、对象键、分片大小、分片数量、状态、过期时间和已上传分片占位 JSON；`/api/v2/files/upload-sessions` 目前只提供创建、查询、取消会话，不承接实际分片内容上传，也不替换旧 `/api/files/upload/**` 生产链路。
|
||||
- 2026-04-08 阶段 3 第二小步补充：上传会话新增完成状态机。`UploadSessionService.completeOwnedSession()` 会复用旧 `FileService.completeUpload()` 完成对象确认、目录补齐、配额/冲突校验和 `FileBlob + StoredFile + FileEntity.VERSION` 双写落库，然后把会话标记为 `COMPLETED`；失败时标记 `FAILED`，过期时标记 `EXPIRED`。当前仍没有独立 v2 分片内容写入端点。
|
||||
- 2026-04-08 阶段 3 第三小步补充：上传会话新增 part 状态记录。`UploadSessionService.recordUploadedPart()` 会校验会话归属、状态、过期时间和 part 范围，把 `etag/size/uploadedAt` 写入 `uploadedPartsJson`，并将新会话推进到 `UPLOADING`。当前实现是会话状态跟踪，不是跨存储驱动的分片内容写入/合并实现。
|
||||
- 2026-04-08 阶段 3 第四小步补充：上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话，尝试删除 `objectKey` 对应的临时 blob，然后标记为 `EXPIRED`。已完成文件不参与清理，避免误删已经落库的生产对象。
|
||||
|
||||
@@ -162,3 +162,4 @@
|
||||
- 2026-04-08 阶段 3 第一小步：新增后端上传会话二期最小骨架，包含 `UploadSession`、`UploadSessionStatus`、`UploadSessionRepository`、`UploadSessionService`，以及受保护的 `/api/v2/files/upload-sessions` 创建、查询、取消接口；旧 `/api/files/upload/**` 上传链路暂不切换，前端上传队列暂不改动。
|
||||
- 2026-04-08 阶段 3 第二小步：新增 `POST /api/v2/files/upload-sessions/{sessionId}/complete`，v2 上传会话可从 `CREATED` 进入 `COMPLETING` 并复用旧 `FileService.completeUpload()` 完成 `FileBlob + StoredFile + FileEntity.VERSION` 落库，成功后标记 `COMPLETED`；取消、失败、过期会话不能完成。实际分片内容上传和前端上传队列仍未切换。
|
||||
- 2026-04-08 阶段 3 第三小步：新增 `PUT /api/v2/files/upload-sessions/{sessionId}/parts/{partIndex}`，用于记录当前用户上传会话的 part 元数据到 `uploadedPartsJson`，并把会话状态从 `CREATED` 推进到 `UPLOADING`；该接口只记录 `etag/size` 等状态，不承担真正的对象存储分片内容写入或合并。
|
||||
- 2026-04-08 阶段 3 第四小步：`UploadSessionService` 新增定时过期清理，按小时扫描 `CREATED/UPLOADING/COMPLETING` 且已过期的会话，尝试删除对应临时 `blobs/...` 对象，并把会话标记为 `EXPIRED`；`COMPLETED/CANCELLED/FAILED/EXPIRED` 不在本轮清理范围内。
|
||||
|
||||
Reference in New Issue
Block a user