refactor(files): reorganize backend package layout
This commit is contained in:
yoyuzh
@@ -4,10 +4,10 @@ import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.admin.AdminMetricsStateRepository;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.FileBlob;
|
||||
import com.yoyuzh.files.FileBlobRepository;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.transfer.OfflineTransferSessionRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -8,12 +8,12 @@ import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.auth.UserRole;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.FileBlobRepository;
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.StoragePolicyRepository;
|
||||
import com.yoyuzh.files.StoragePolicyService;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicyRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.transfer.OfflineTransferSessionRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -4,9 +4,9 @@ import com.yoyuzh.api.v2.ApiV2ExceptionHandler;
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.PageResponse;
|
||||
import com.yoyuzh.files.FileMetadataResponse;
|
||||
import com.yoyuzh.files.FileSearchQuery;
|
||||
import com.yoyuzh.files.FileSearchService;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.search.FileSearchQuery;
|
||||
import com.yoyuzh.files.search.FileSearchService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.MethodParameter;
|
||||
|
||||
@@ -2,9 +2,9 @@ package com.yoyuzh.api.v2.files;
|
||||
|
||||
import com.yoyuzh.auth.CustomUserDetailsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.UploadSession;
|
||||
import com.yoyuzh.files.UploadSessionService;
|
||||
import com.yoyuzh.files.UploadSessionStatus;
|
||||
import com.yoyuzh.files.upload.UploadSession;
|
||||
import com.yoyuzh.files.upload.UploadSessionService;
|
||||
import com.yoyuzh.files.upload.UploadSessionStatus;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.MediaType;
|
||||
@@ -19,6 +19,7 @@ import org.springframework.web.method.support.HandlerMethodArgumentResolver;
|
||||
import org.springframework.web.method.support.ModelAndViewContainer;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
@@ -69,6 +70,7 @@ class UploadSessionV2ControllerTest {
|
||||
.andExpect(jsonPath("$.data.sessionId").value("session-1"))
|
||||
.andExpect(jsonPath("$.data.objectKey").value("blobs/session-1"))
|
||||
.andExpect(jsonPath("$.data.status").value("CREATED"))
|
||||
.andExpect(jsonPath("$.data.multipartUpload").value(true))
|
||||
.andExpect(jsonPath("$.data.chunkSize").value(8388608))
|
||||
.andExpect(jsonPath("$.data.chunkCount").value(3));
|
||||
}
|
||||
@@ -85,7 +87,8 @@ class UploadSessionV2ControllerTest {
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value(0))
|
||||
.andExpect(jsonPath("$.data.sessionId").value("session-1"))
|
||||
.andExpect(jsonPath("$.data.status").value("CREATED"));
|
||||
.andExpect(jsonPath("$.data.status").value("CREATED"))
|
||||
.andExpect(jsonPath("$.data.multipartUpload").value(true));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -127,6 +130,29 @@ class UploadSessionV2ControllerTest {
|
||||
.andExpect(jsonPath("$.data.status").value("UPLOADING"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPrepareMultipartPartUploadWithV2Envelope() throws Exception {
|
||||
User user = createUser(7L);
|
||||
when(userDetailsService.loadDomainUser("alice")).thenReturn(user);
|
||||
when(uploadSessionService.prepareOwnedPartUpload(user, "session-1", 1))
|
||||
.thenReturn(new com.yoyuzh.files.storage.PreparedUpload(
|
||||
true,
|
||||
"https://upload.example.com/session-1/part-2",
|
||||
"PUT",
|
||||
Map.of("Content-Type", "video/mp4"),
|
||||
"blobs/session-1"
|
||||
));
|
||||
|
||||
mockMvc.perform(get("/api/v2/files/upload-sessions/session-1/parts/1/prepare")
|
||||
.with(user(userDetails())))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value(0))
|
||||
.andExpect(jsonPath("$.data.direct").value(true))
|
||||
.andExpect(jsonPath("$.data.uploadUrl").value("https://upload.example.com/session-1/part-2"))
|
||||
.andExpect(jsonPath("$.data.method").value("PUT"))
|
||||
.andExpect(jsonPath("$.data.headers['Content-Type']").value("video/mp4"));
|
||||
}
|
||||
|
||||
private UserDetails userDetails() {
|
||||
return org.springframework.security.core.userdetails.User
|
||||
.withUsername("alice")
|
||||
@@ -172,6 +198,7 @@ class UploadSessionV2ControllerTest {
|
||||
session.setContentType("video/mp4");
|
||||
session.setSize(20L * 1024 * 1024);
|
||||
session.setObjectKey("blobs/session-1");
|
||||
session.setMultipartUploadId("upload-123");
|
||||
session.setChunkSize(8L * 1024 * 1024);
|
||||
session.setChunkCount(3);
|
||||
session.setStatus(UploadSessionStatus.CREATED);
|
||||
|
||||
@@ -4,12 +4,12 @@ import com.jayway.jsonpath.JsonPath;
|
||||
import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.FileBlob;
|
||||
import com.yoyuzh.files.FileBlobRepository;
|
||||
import com.yoyuzh.files.FileShareLink;
|
||||
import com.yoyuzh.files.FileShareLinkRepository;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.share.FileShareLink;
|
||||
import com.yoyuzh.files.share.FileShareLinkRepository;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -30,6 +30,8 @@ import static org.springframework.security.test.web.servlet.request.SecurityMock
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
@@ -274,6 +276,120 @@ class ShareV2ControllerIntegrationTest {
|
||||
.andExpect(jsonPath("$.code").value(2404));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDownloadSharedFileAndCountQuota() throws Exception {
|
||||
String createResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"maxDownloads": 1,
|
||||
"allowDownload": true
|
||||
}
|
||||
""".formatted(sharedFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
String token = JsonPath.read(createResponse, "$.data.token");
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", token)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(header().string("Content-Disposition", "attachment; filename*=UTF-8''notes.txt"))
|
||||
.andExpect(content().contentType("text/plain"))
|
||||
.andExpect(content().bytes("hello".getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", token)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isForbidden())
|
||||
.andExpect(jsonPath("$.code").value(2403));
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", token).with(anonymous()))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.downloadCount").value(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectDownloadWhenSharePolicyBlocksIt() throws Exception {
|
||||
String passwordResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"password": "Share123",
|
||||
"allowDownload": true
|
||||
}
|
||||
""".formatted(sharedFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
String passwordToken = JsonPath.read(passwordResponse, "$.data.token");
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", passwordToken)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isBadRequest())
|
||||
.andExpect(jsonPath("$.code").value(2400));
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", passwordToken)
|
||||
.with(anonymous())
|
||||
.param("download", "1")
|
||||
.param("password", "Share123"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(content().bytes("hello".getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
String disabledResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"allowDownload": false
|
||||
}
|
||||
""".formatted(sharedFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
String disabledToken = JsonPath.read(disabledResponse, "$.data.token");
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", disabledToken)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isForbidden())
|
||||
.andExpect(jsonPath("$.code").value(2403));
|
||||
|
||||
String expiredResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"allowDownload": true
|
||||
}
|
||||
""".formatted(sharedFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
String expiredToken = JsonPath.read(expiredResponse, "$.data.token");
|
||||
FileShareLink expiredShare = fileShareLinkRepository.findByToken(expiredToken).orElseThrow();
|
||||
expiredShare.setExpiresAt(LocalDateTime.now().minusMinutes(1));
|
||||
fileShareLinkRepository.save(expiredShare);
|
||||
|
||||
mockMvc.perform(get("/api/v2/shares/{token}", expiredToken)
|
||||
.with(anonymous())
|
||||
.param("download", "1"))
|
||||
.andExpect(status().isNotFound())
|
||||
.andExpect(jsonPath("$.code").value(2404));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDenyDeletingOtherUsersShare() throws Exception {
|
||||
String createResponse = mockMvc.perform(post("/api/v2/shares")
|
||||
|
||||
@@ -1,16 +1,22 @@
|
||||
package com.yoyuzh.api.v2.tasks;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.BackgroundTask;
|
||||
import com.yoyuzh.files.BackgroundTaskRepository;
|
||||
import com.yoyuzh.files.BackgroundTaskStatus;
|
||||
import com.yoyuzh.files.BackgroundTaskType;
|
||||
import com.yoyuzh.files.StoredFile;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.tasks.BackgroundTask;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskRepository;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskStatus;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskStartupRecovery;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskType;
|
||||
import com.yoyuzh.files.tasks.BackgroundTaskWorker;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -19,10 +25,17 @@ import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.not;
|
||||
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.anonymous;
|
||||
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
@@ -55,23 +68,39 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
@Autowired
|
||||
private BackgroundTaskRepository backgroundTaskRepository;
|
||||
|
||||
@Autowired
|
||||
private BackgroundTaskWorker backgroundTaskWorker;
|
||||
|
||||
@Autowired
|
||||
private BackgroundTaskStartupRecovery backgroundTaskStartupRecovery;
|
||||
|
||||
@Autowired
|
||||
private FileBlobRepository fileBlobRepository;
|
||||
|
||||
@Autowired
|
||||
private FileContentStorage fileContentStorage;
|
||||
|
||||
@Autowired
|
||||
private StoredFileRepository storedFileRepository;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
private Long aliceId;
|
||||
private Long archiveDirectoryId;
|
||||
private Long archiveFileId;
|
||||
private Long extractFileId;
|
||||
private Long invalidExtractFileId;
|
||||
private Long unsupportedExtractFileId;
|
||||
private Long mediaFileId;
|
||||
private Long foreignFileId;
|
||||
private Long deletedFileId;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
void setUp() throws Exception {
|
||||
backgroundTaskRepository.deleteAll();
|
||||
storedFileRepository.deleteAll();
|
||||
fileBlobRepository.deleteAll();
|
||||
userRepository.deleteAll();
|
||||
|
||||
User alice = new User();
|
||||
@@ -80,7 +109,8 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
alice.setPhoneNumber("13800138000");
|
||||
alice.setPasswordHash("encoded-password");
|
||||
alice.setCreatedAt(LocalDateTime.now());
|
||||
userRepository.save(alice);
|
||||
alice = userRepository.save(alice);
|
||||
aliceId = alice.getId();
|
||||
|
||||
User bob = new User();
|
||||
bob.setUsername("bob");
|
||||
@@ -91,11 +121,62 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
bob = userRepository.save(bob);
|
||||
|
||||
archiveDirectoryId = storedFileRepository.save(createFile(alice, "/docs", "archive", true, null, 0L, null)).getId();
|
||||
archiveFileId = storedFileRepository.save(createFile(alice, "/docs", "archive-source.txt", false, "text/plain", 12L, null)).getId();
|
||||
extractFileId = storedFileRepository.save(createFile(alice, "/docs", "extract.zip", false, "application/zip", 32L, null)).getId();
|
||||
storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs/archive",
|
||||
"nested.txt",
|
||||
"text/plain",
|
||||
"archive-nested",
|
||||
"nested-content".getBytes(StandardCharsets.UTF_8)
|
||||
));
|
||||
archiveFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs",
|
||||
"archive-source.txt",
|
||||
"text/plain",
|
||||
"archive-source",
|
||||
"archive-source".getBytes(StandardCharsets.UTF_8)
|
||||
)).getId();
|
||||
extractFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs",
|
||||
"extract.zip",
|
||||
"application/zip",
|
||||
"extract-source",
|
||||
createZipArchive(Map.of(
|
||||
"extract/", "",
|
||||
"extract/nested/", "",
|
||||
"extract/notes.txt", "hello",
|
||||
"extract/nested/todo.txt", "world"
|
||||
))
|
||||
)).getId();
|
||||
invalidExtractFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs",
|
||||
"broken.zip",
|
||||
"application/zip",
|
||||
"broken-extract",
|
||||
"not-a-zip".getBytes(StandardCharsets.UTF_8)
|
||||
)).getId();
|
||||
unsupportedExtractFileId = storedFileRepository.save(createFile(alice, "/docs", "backup.7z", false, "application/x-7z-compressed", 64L, null)).getId();
|
||||
mediaFileId = storedFileRepository.save(createFile(alice, "/docs", "media.png", false, "image/png", 24L, null)).getId();
|
||||
foreignFileId = storedFileRepository.save(createFile(bob, "/docs", "foreign.zip", false, "application/zip", 32L, null)).getId();
|
||||
deletedFileId = storedFileRepository.save(createFile(alice, "/docs", "deleted.zip", false, "application/zip", 32L, LocalDateTime.now())).getId();
|
||||
foreignFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
bob,
|
||||
"/docs",
|
||||
"foreign.zip",
|
||||
"application/zip",
|
||||
"foreign-zip",
|
||||
createZipArchive(Map.of("foreign.txt", "blocked"))
|
||||
)).getId();
|
||||
deletedFileId = storedFileRepository.save(createBlobBackedFile(
|
||||
alice,
|
||||
"/docs",
|
||||
"deleted.zip",
|
||||
"application/zip",
|
||||
"deleted-zip",
|
||||
createZipArchive(Map.of("deleted.txt", "gone")),
|
||||
LocalDateTime.now()
|
||||
)).getId();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -132,6 +213,9 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"fileId\":" + archiveDirectoryId)))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"path\":\"/docs/archive\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"directory\":true")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":4")))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
@@ -148,6 +232,11 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.type").value("EXTRACT"))
|
||||
.andExpect(jsonPath("$.data.status").value("QUEUED"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputPath\":\"/docs\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputDirectoryName\":\"extract\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
@@ -164,6 +253,9 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.type").value("MEDIA_META"))
|
||||
.andExpect(jsonPath("$.data.status").value("QUEUED"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":0")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":2")))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
@@ -187,11 +279,13 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
|
||||
mockMvc.perform(delete("/api/v2/tasks/{id}", extractId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("CANCELLED"));
|
||||
.andExpect(jsonPath("$.data.status").value("CANCELLED"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"cancelled\"")));
|
||||
|
||||
BackgroundTask cancelled = backgroundTaskRepository.findById(extractId).orElseThrow();
|
||||
assertThat(cancelled.getStatus()).isEqualTo(BackgroundTaskStatus.CANCELLED);
|
||||
assertThat(cancelled.getFinishedAt()).isNotNull();
|
||||
assertThat(cancelled.getPublicStateJson()).contains("\"phase\":\"cancelled\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -216,6 +310,289 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
|
||||
mockMvc.perform(delete("/api/v2/tasks/{id}", taskId).with(user("bob")))
|
||||
.andExpect(status().isNotFound());
|
||||
|
||||
mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("bob")))
|
||||
.andExpect(status().isNotFound());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectExtractTaskForNonZipCompatibleArchive() throws Exception {
|
||||
mockMvc.perform(post("/api/v2/tasks/extract")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/backup.7z"
|
||||
}
|
||||
""".formatted(unsupportedExtractFileId)))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCompleteArchiveTaskThroughWorkerAndExposeTerminalState() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/archive")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/archive-source.txt"
|
||||
}
|
||||
""".formatted(archiveFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
String taskResponse = mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("COMPLETED"))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
|
||||
Map<String, Object> publicState = readPublicState(taskResponse);
|
||||
assertThat(publicState).containsEntry("worker", "archive");
|
||||
assertThat(publicState).containsEntry("archivedFilename", "archive-source.txt.zip");
|
||||
assertThat(publicState).containsEntry("archivedPath", "/docs");
|
||||
assertThat(publicState).containsEntry("phase", "completed");
|
||||
assertThat(publicState).containsEntry("attemptCount", 1);
|
||||
assertThat(publicState).containsEntry("maxAttempts", 4);
|
||||
assertThat(publicState).containsEntry("processedFileCount", 1);
|
||||
assertThat(publicState).containsEntry("totalFileCount", 1);
|
||||
assertThat(publicState).containsEntry("processedDirectoryCount", 0);
|
||||
assertThat(publicState).containsEntry("totalDirectoryCount", 0);
|
||||
assertThat(publicState).containsEntry("progressPercent", 100);
|
||||
assertThat(publicState.get("heartbeatAt")).isNotNull();
|
||||
assertThat(publicState).doesNotContainKey("workerOwner");
|
||||
assertThat(publicState).doesNotContainKey("leaseExpiresAt");
|
||||
assertThat(publicState.get("archivedFileId")).isNotNull();
|
||||
assertThat(publicState.get("archiveSize")).isNotNull();
|
||||
|
||||
BackgroundTask completed = backgroundTaskRepository.findById(taskId).orElseThrow();
|
||||
assertThat(completed.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
|
||||
assertThat(completed.getFinishedAt()).isNotNull();
|
||||
assertThat(completed.getErrorMessage()).isNull();
|
||||
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs", "archive-source.txt.zip")).isPresent();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCompleteExtractTaskThroughWorkerAndExposeTerminalState() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/extract")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/extract.zip"
|
||||
}
|
||||
""".formatted(extractFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
String taskResponse = mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("COMPLETED"))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
|
||||
Map<String, Object> publicState = readPublicState(taskResponse);
|
||||
assertThat(publicState).containsEntry("worker", "extract");
|
||||
assertThat(publicState).containsEntry("extractedPath", "/docs/extract");
|
||||
assertThat(publicState).containsEntry("extractedFileCount", 2);
|
||||
assertThat(publicState).containsEntry("extractedDirectoryCount", 2);
|
||||
assertThat(publicState).containsEntry("phase", "completed");
|
||||
assertThat(publicState).containsEntry("attemptCount", 1);
|
||||
assertThat(publicState).containsEntry("maxAttempts", 3);
|
||||
assertThat(publicState).containsEntry("processedFileCount", 2);
|
||||
assertThat(publicState).containsEntry("totalFileCount", 2);
|
||||
assertThat(publicState).containsEntry("processedDirectoryCount", 2);
|
||||
assertThat(publicState).containsEntry("totalDirectoryCount", 2);
|
||||
assertThat(publicState).containsEntry("progressPercent", 100);
|
||||
assertThat(publicState.get("heartbeatAt")).isNotNull();
|
||||
assertThat(publicState).doesNotContainKey("workerOwner");
|
||||
assertThat(publicState).doesNotContainKey("leaseExpiresAt");
|
||||
|
||||
BackgroundTask completed = backgroundTaskRepository.findById(taskId).orElseThrow();
|
||||
assertThat(completed.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
|
||||
assertThat(completed.getFinishedAt()).isNotNull();
|
||||
assertThat(completed.getErrorMessage()).isNull();
|
||||
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs/extract", "notes.txt")).isPresent();
|
||||
assertThat(storedFileRepository.findByUserIdAndPathAndFilename(aliceId, "/docs/extract/nested", "todo.txt")).isPresent();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldMarkExtractTaskFailedWhenWorkerHitsInvalidArchiveContent() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/extract")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/broken.zip"
|
||||
}
|
||||
""".formatted(invalidExtractFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
mockMvc.perform(get("/api/v2/tasks/{id}", taskId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("FAILED"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"failed\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":1")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"failureCategory\":\"DATA_STATE\"")))
|
||||
.andExpect(jsonPath("$.data.errorMessage").value("extract task only supports zip-compatible archives"));
|
||||
|
||||
BackgroundTask failed = backgroundTaskRepository.findById(taskId).orElseThrow();
|
||||
assertThat(failed.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED);
|
||||
assertThat(failed.getFinishedAt()).isNotNull();
|
||||
assertThat(failed.getErrorMessage()).isEqualTo("extract task only supports zip-compatible archives");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRetryFailedTaskAndResetStateToQueued() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/extract")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/broken.zip"
|
||||
}
|
||||
""".formatted(invalidExtractFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
String retryResponse = mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("QUEUED"))
|
||||
.andExpect(jsonPath("$.data.errorMessage").doesNotExist())
|
||||
.andExpect(jsonPath("$.data.finishedAt").doesNotExist())
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
|
||||
Map<String, Object> publicState = readPublicState(retryResponse);
|
||||
assertThat(publicState).containsEntry("phase", "queued");
|
||||
assertThat(publicState).containsEntry("outputPath", "/docs");
|
||||
assertThat(publicState).containsEntry("outputDirectoryName", "broken");
|
||||
assertThat(publicState).containsEntry("attemptCount", 0);
|
||||
assertThat(publicState).containsEntry("maxAttempts", 3);
|
||||
assertThat(publicState).doesNotContainKey("worker");
|
||||
assertThat(publicState).doesNotContainKey("processedFileCount");
|
||||
assertThat(publicState).doesNotContainKey("totalFileCount");
|
||||
|
||||
BackgroundTask retried = backgroundTaskRepository.findById(taskId).orElseThrow();
|
||||
assertThat(retried.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED);
|
||||
assertThat(retried.getFinishedAt()).isNull();
|
||||
assertThat(retried.getErrorMessage()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectRetryForNonFailedTask() throws Exception {
|
||||
String response = mockMvc.perform(post("/api/v2/tasks/archive")
|
||||
.with(user("alice"))
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"fileId": %d,
|
||||
"path": "/docs/archive-source.txt"
|
||||
}
|
||||
""".formatted(archiveFileId)))
|
||||
.andExpect(status().isOk())
|
||||
.andReturn()
|
||||
.getResponse()
|
||||
.getContentAsString();
|
||||
Long taskId = ((Number) JsonPath.read(response, "$.data.id")).longValue();
|
||||
|
||||
mockMvc.perform(post("/api/v2/tasks/{id}/retry", taskId).with(user("alice")))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRecoverOnlyExpiredRunningTaskBackToQueuedOnStartup() throws Exception {
|
||||
BackgroundTask expired = new BackgroundTask();
|
||||
expired.setType(BackgroundTaskType.EXTRACT);
|
||||
expired.setStatus(BackgroundTaskStatus.RUNNING);
|
||||
expired.setUserId(aliceId);
|
||||
expired.setCorrelationId("recover-1");
|
||||
expired.setAttemptCount(1);
|
||||
expired.setMaxAttempts(3);
|
||||
expired.setLeaseOwner("worker-stale");
|
||||
expired.setLeaseExpiresAt(LocalDateTime.now().minusMinutes(2));
|
||||
expired.setHeartbeatAt(LocalDateTime.now().minusMinutes(3));
|
||||
expired.setPublicStateJson("""
|
||||
{"fileId":%d,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-stale","attemptCount":1,"maxAttempts":3}
|
||||
""".formatted(extractFileId));
|
||||
expired.setPrivateStateJson("""
|
||||
{"fileId":%d,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"}
|
||||
""".formatted(extractFileId));
|
||||
expired.setErrorMessage("stale worker");
|
||||
expired.setFinishedAt(LocalDateTime.now());
|
||||
expired = backgroundTaskRepository.save(expired);
|
||||
|
||||
BackgroundTask fresh = new BackgroundTask();
|
||||
fresh.setType(BackgroundTaskType.EXTRACT);
|
||||
fresh.setStatus(BackgroundTaskStatus.RUNNING);
|
||||
fresh.setUserId(aliceId);
|
||||
fresh.setCorrelationId("recover-2");
|
||||
fresh.setAttemptCount(1);
|
||||
fresh.setMaxAttempts(3);
|
||||
fresh.setLeaseOwner("worker-live");
|
||||
fresh.setLeaseExpiresAt(LocalDateTime.now().plusMinutes(5));
|
||||
fresh.setHeartbeatAt(LocalDateTime.now());
|
||||
fresh.setPublicStateJson("""
|
||||
{"fileId":%d,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-live","attemptCount":1,"maxAttempts":3}
|
||||
""".formatted(extractFileId));
|
||||
fresh.setPrivateStateJson("""
|
||||
{"fileId":%d,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"}
|
||||
""".formatted(extractFileId));
|
||||
fresh = backgroundTaskRepository.save(fresh);
|
||||
|
||||
backgroundTaskStartupRecovery.recoverOnStartup();
|
||||
|
||||
mockMvc.perform(get("/api/v2/tasks/{id}", expired.getId()).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("QUEUED"))
|
||||
.andExpect(jsonPath("$.data.errorMessage").doesNotExist())
|
||||
.andExpect(jsonPath("$.data.finishedAt").doesNotExist())
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"phase\":\"queued\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"attemptCount\":1")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"maxAttempts\":3")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputPath\":\"/docs\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"outputDirectoryName\":\"extract\"")))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"worker\""))))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"workerOwner\""))))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", not(containsString("\"leaseExpiresAt\""))));
|
||||
|
||||
mockMvc.perform(get("/api/v2/tasks/{id}", fresh.getId()).with(user("alice")))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.status").value("RUNNING"))
|
||||
.andExpect(jsonPath("$.data.publicStateJson", containsString("\"workerOwner\":\"worker-live\"")));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -274,4 +651,54 @@ class BackgroundTaskV2ControllerIntegrationTest {
|
||||
file.setDeletedAt(deletedAt);
|
||||
return file;
|
||||
}
|
||||
|
||||
private StoredFile createBlobBackedFile(User user,
|
||||
String path,
|
||||
String filename,
|
||||
String contentType,
|
||||
String objectKeySuffix,
|
||||
byte[] content) {
|
||||
return createBlobBackedFile(user, path, filename, contentType, objectKeySuffix, content, null);
|
||||
}
|
||||
|
||||
private StoredFile createBlobBackedFile(User user,
|
||||
String path,
|
||||
String filename,
|
||||
String contentType,
|
||||
String objectKeySuffix,
|
||||
byte[] content,
|
||||
LocalDateTime deletedAt) {
|
||||
String objectKey = "blobs/test-background-task/" + objectKeySuffix;
|
||||
fileContentStorage.storeBlob(objectKey, contentType, content);
|
||||
|
||||
FileBlob blob = new FileBlob();
|
||||
blob.setObjectKey(objectKey);
|
||||
blob.setContentType(contentType);
|
||||
blob.setSize((long) content.length);
|
||||
blob = fileBlobRepository.save(blob);
|
||||
|
||||
StoredFile file = createFile(user, path, filename, false, contentType, (long) content.length, deletedAt);
|
||||
file.setBlob(blob);
|
||||
return file;
|
||||
}
|
||||
|
||||
private byte[] createZipArchive(Map<String, String> entries) throws IOException {
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) {
|
||||
for (Map.Entry<String, String> entry : entries.entrySet()) {
|
||||
zipOutputStream.putNextEntry(new ZipEntry(entry.getKey()));
|
||||
if (!entry.getKey().endsWith("/")) {
|
||||
zipOutputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
zipOutputStream.closeEntry();
|
||||
}
|
||||
}
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
|
||||
private Map<String, Object> readPublicState(String taskResponse) throws Exception {
|
||||
String publicStateJson = JsonPath.read(taskResponse, "$.data.publicStateJson");
|
||||
return objectMapper.readValue(publicStateJson, new TypeReference<Map<String, Object>>() {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@ import com.yoyuzh.auth.dto.UpdateUserAvatarRequest;
|
||||
import com.yoyuzh.auth.dto.UpdateUserPasswordRequest;
|
||||
import com.yoyuzh.auth.dto.UpdateUserProfileRequest;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.InitiateUploadResponse;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.upload.InitiateUploadResponse;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import com.yoyuzh.files.storage.PreparedUpload;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -39,7 +39,7 @@ class AuthSingleDeviceIntegrationTest {
|
||||
private UserRepository userRepository;
|
||||
|
||||
@Autowired
|
||||
private com.yoyuzh.files.StoredFileRepository storedFileRepository;
|
||||
private com.yoyuzh.files.core.StoredFileRepository storedFileRepository;
|
||||
|
||||
@Autowired
|
||||
private RefreshTokenRepository refreshTokenRepository;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.yoyuzh.auth;
|
||||
|
||||
import com.yoyuzh.files.FileService;
|
||||
import com.yoyuzh.files.StoredFileRepository;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
|
||||
@@ -1,246 +0,0 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.api.v2.ApiV2Exception;
|
||||
import com.yoyuzh.auth.User;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BackgroundTaskServiceTest {
|
||||
|
||||
@Mock
|
||||
private BackgroundTaskRepository backgroundTaskRepository;
|
||||
|
||||
@Mock
|
||||
private StoredFileRepository storedFileRepository;
|
||||
|
||||
private BackgroundTaskService backgroundTaskService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
backgroundTaskService = new BackgroundTaskService(backgroundTaskRepository, storedFileRepository, new ObjectMapper());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationForForeignFile() {
|
||||
User user = createUser(7L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(99L, 7L)).thenReturn(Optional.empty());
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
99L,
|
||||
"/docs/foreign.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("file not found");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationForDeletedFile() {
|
||||
User user = createUser(7L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(100L, 7L)).thenReturn(Optional.empty());
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
100L,
|
||||
"/docs/deleted.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("file not found");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationWhenRequestedPathDoesNotMatchFile() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(11L, user, "/docs", "real.txt", false, "text/plain", 3L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(file));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
11L,
|
||||
"/docs/fake.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("task path does not match file path");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectExtractTaskForDirectory() {
|
||||
User user = createUser(7L);
|
||||
StoredFile directory = createStoredFile(12L, user, "/", "bundle", true, null, 0L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(12L, 7L)).thenReturn(Optional.of(directory));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.EXTRACT,
|
||||
12L,
|
||||
"/bundle",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("task target type is not supported");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectMediaMetadataTaskForNonMediaFile() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(13L, user, "/docs", "notes.txt", false, "text/plain", 9L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(13L, 7L)).thenReturn(Optional.of(file));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.MEDIA_META,
|
||||
13L,
|
||||
"/docs/notes.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("media metadata task only supports media files");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCreateTaskStateFromServerFilePath() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(14L, user, "/docs", "photo.png", false, "image/png", 15L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(14L, 7L)).thenReturn(Optional.of(file));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask task = backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.MEDIA_META,
|
||||
14L,
|
||||
"/docs/photo.png",
|
||||
"media-1"
|
||||
);
|
||||
|
||||
assertThat(task.getPublicStateJson()).contains("\"fileId\":14");
|
||||
assertThat(task.getPublicStateJson()).contains("\"path\":\"/docs/photo.png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"filename\":\"photo.png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"directory\":false");
|
||||
assertThat(task.getPublicStateJson()).contains("\"contentType\":\"image/png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"size\":15");
|
||||
assertThat(task.getPrivateStateJson()).contains("\"taskType\":\"MEDIA_META\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldClaimQueuedTaskOnlyWhenRepositoryTransitionSucceeds() {
|
||||
BackgroundTask task = createTask(1L, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskRepository.claimQueuedTask(
|
||||
eq(1L),
|
||||
eq(BackgroundTaskStatus.QUEUED),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(1L)).thenReturn(Optional.of(task));
|
||||
|
||||
Optional<BackgroundTask> result = backgroundTaskService.claimQueuedTask(1L);
|
||||
|
||||
assertThat(result).containsSame(task);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldNotClaimTaskWhenRepositoryTransitionWasSkipped() {
|
||||
when(backgroundTaskRepository.claimQueuedTask(
|
||||
eq(2L),
|
||||
eq(BackgroundTaskStatus.QUEUED),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
any()
|
||||
)).thenReturn(0);
|
||||
|
||||
Optional<BackgroundTask> result = backgroundTaskService.claimQueuedTask(2L);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCompleteRunningWorkerTaskAndMergePublicState() {
|
||||
BackgroundTask task = createTask(3L, BackgroundTaskStatus.RUNNING);
|
||||
task.setPublicStateJson("{\"fileId\":11}");
|
||||
when(backgroundTaskRepository.findById(3L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskCompleted(3L, Map.of("worker", "noop"));
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
|
||||
assertThat(result.getFinishedAt()).isNotNull();
|
||||
assertThat(result.getErrorMessage()).isNull();
|
||||
assertThat(result.getPublicStateJson()).contains("\"fileId\":11");
|
||||
assertThat(result.getPublicStateJson()).contains("\"worker\":\"noop\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRecordWorkerFailureMessage() {
|
||||
BackgroundTask task = createTask(4L, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskRepository.findById(4L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskFailed(4L, "media parser unavailable");
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED);
|
||||
assertThat(result.getFinishedAt()).isNotNull();
|
||||
assertThat(result.getErrorMessage()).isEqualTo("media parser unavailable");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFindQueuedTaskIdsInCreatedOrderLimit() {
|
||||
BackgroundTask first = createTask(5L, BackgroundTaskStatus.QUEUED);
|
||||
BackgroundTask second = createTask(6L, BackgroundTaskStatus.QUEUED);
|
||||
when(backgroundTaskRepository.findByStatusOrderByCreatedAtAsc(eq(BackgroundTaskStatus.QUEUED), any()))
|
||||
.thenReturn(List.of(first, second));
|
||||
|
||||
List<Long> result = backgroundTaskService.findQueuedTaskIds(2);
|
||||
|
||||
assertThat(result).containsExactly(5L, 6L);
|
||||
}
|
||||
|
||||
private BackgroundTask createTask(Long id, BackgroundTaskStatus status) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setId(id);
|
||||
task.setType(BackgroundTaskType.MEDIA_META);
|
||||
task.setStatus(status);
|
||||
task.setUserId(7L);
|
||||
task.setPublicStateJson("{}");
|
||||
task.setPrivateStateJson("{}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private User createUser(Long id) {
|
||||
User user = new User();
|
||||
user.setId(id);
|
||||
user.setUsername("alice");
|
||||
return user;
|
||||
}
|
||||
|
||||
private StoredFile createStoredFile(Long id,
|
||||
User user,
|
||||
String path,
|
||||
String filename,
|
||||
boolean directory,
|
||||
String contentType,
|
||||
Long size) {
|
||||
StoredFile file = new StoredFile();
|
||||
file.setId(id);
|
||||
file.setUser(user);
|
||||
file.setPath(path);
|
||||
file.setFilename(filename);
|
||||
file.setDirectory(directory);
|
||||
file.setContentType(contentType);
|
||||
file.setSize(size);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BackgroundTaskWorkerTest {
|
||||
|
||||
@Mock
|
||||
private BackgroundTaskService backgroundTaskService;
|
||||
@Mock
|
||||
private BackgroundTaskHandler backgroundTaskHandler;
|
||||
|
||||
private BackgroundTaskWorker backgroundTaskWorker;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
backgroundTaskWorker = new BackgroundTaskWorker(backgroundTaskService, List.of(backgroundTaskHandler));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldClaimAndCompleteQueuedTaskThroughNoopHandler() {
|
||||
BackgroundTask task = createTask(1L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L));
|
||||
when(backgroundTaskService.claimQueuedTask(1L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskHandler.supports(BackgroundTaskType.ARCHIVE)).thenReturn(true);
|
||||
when(backgroundTaskHandler.handle(task)).thenReturn(new BackgroundTaskHandlerResult(Map.of("worker", "noop")));
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
verify(backgroundTaskHandler).handle(task);
|
||||
verify(backgroundTaskService).markWorkerTaskCompleted(1L, Map.of("worker", "noop"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldSkipTaskThatWasNotClaimed() {
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L));
|
||||
when(backgroundTaskService.claimQueuedTask(1L)).thenReturn(Optional.empty());
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isZero();
|
||||
verify(backgroundTaskHandler, never()).handle(org.mockito.ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldMarkTaskFailedWhenHandlerThrows() {
|
||||
BackgroundTask task = createTask(2L, BackgroundTaskType.MEDIA_META, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(2L));
|
||||
when(backgroundTaskService.claimQueuedTask(2L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskHandler.supports(BackgroundTaskType.MEDIA_META)).thenReturn(true);
|
||||
when(backgroundTaskHandler.handle(task)).thenThrow(new IllegalStateException("media parser unavailable"));
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
verify(backgroundTaskService).markWorkerTaskFailed(2L, "media parser unavailable");
|
||||
}
|
||||
|
||||
private BackgroundTask createTask(Long id, BackgroundTaskType type, BackgroundTaskStatus status) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setId(id);
|
||||
task.setType(type);
|
||||
task.setStatus(status);
|
||||
task.setUserId(7L);
|
||||
task.setPublicStateJson("{}");
|
||||
task.setPrivateStateJson("{}");
|
||||
return task;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
@@ -1,6 +1,10 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.policy.StoragePolicy;
|
||||
import com.yoyuzh.files.policy.StoragePolicyCredentialMode;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.files.policy.StoragePolicyType;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -1,9 +1,11 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.admin.AdminMetricsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.share.FileShareLinkRepository;
|
||||
import com.yoyuzh.files.upload.InitiateUploadRequest;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -1,9 +1,19 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.admin.AdminMetricsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.policy.StoragePolicy;
|
||||
import com.yoyuzh.files.policy.StoragePolicyCredentialMode;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.files.policy.StoragePolicyType;
|
||||
import com.yoyuzh.files.share.CreateFileShareLinkResponse;
|
||||
import com.yoyuzh.files.share.FileShareLink;
|
||||
import com.yoyuzh.files.share.FileShareLinkRepository;
|
||||
import com.yoyuzh.files.upload.CompleteUploadRequest;
|
||||
import com.yoyuzh.files.upload.InitiateUploadRequest;
|
||||
import com.yoyuzh.files.upload.InitiateUploadResponse;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import com.yoyuzh.files.storage.PreparedUpload;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -19,6 +29,8 @@ import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Clock;
|
||||
@@ -29,7 +41,10 @@ import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
@@ -722,6 +737,138 @@ class FileServiceTest {
|
||||
verify(fileContentStorage).readBlob("blobs/blob-13");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldBuildZipBytesForDirectoryForBackgroundArchiveReuse() throws Exception {
|
||||
User user = createUser(7L);
|
||||
StoredFile directory = createDirectory(10L, user, "/docs", "archive");
|
||||
StoredFile childDirectory = createDirectory(11L, user, "/docs/archive", "nested");
|
||||
StoredFile childFile = createFile(12L, user, "/docs/archive", "notes.txt");
|
||||
StoredFile nestedFile = createFile(13L, user, "/docs/archive/nested", "todo.txt");
|
||||
|
||||
when(storedFileRepository.findByUserIdAndPathEqualsOrDescendant(7L, "/docs/archive"))
|
||||
.thenReturn(List.of(childDirectory, childFile, nestedFile));
|
||||
when(fileContentStorage.readBlob("blobs/blob-12"))
|
||||
.thenReturn("hello".getBytes(StandardCharsets.UTF_8));
|
||||
when(fileContentStorage.readBlob("blobs/blob-13"))
|
||||
.thenReturn("world".getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
byte[] archiveBytes = fileService.buildArchiveBytes(directory);
|
||||
|
||||
Map<String, String> entries = readZipEntries(archiveBytes);
|
||||
|
||||
assertThat(entries).containsEntry("archive/", "");
|
||||
assertThat(entries).containsEntry("archive/nested/", "");
|
||||
assertThat(entries).containsEntry("archive/notes.txt", "hello");
|
||||
assertThat(entries).containsEntry("archive/nested/todo.txt", "world");
|
||||
verify(fileContentStorage).readBlob("blobs/blob-12");
|
||||
verify(fileContentStorage).readBlob("blobs/blob-13");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldBuildZipBytesForSingleFileForBackgroundArchiveReuse() throws Exception {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createFile(12L, user, "/docs", "notes.txt");
|
||||
when(fileContentStorage.readBlob("blobs/blob-12"))
|
||||
.thenReturn("hello".getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
byte[] archiveBytes = fileService.buildArchiveBytes(file);
|
||||
|
||||
Map<String, String> entries = readZipEntries(archiveBytes);
|
||||
|
||||
assertThat(entries).containsEntry("notes.txt", "hello");
|
||||
verify(fileContentStorage).readBlob("blobs/blob-12");
|
||||
verify(storedFileRepository, never()).findByUserIdAndPathEqualsOrDescendant(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReadZipCompatibleArchiveForExtractTaskReuse() throws Exception {
|
||||
User user = createUser(7L);
|
||||
StoredFile archive = createFile(20L, user, "/docs", "extract.zip", createBlob(20L, "blobs/blob-20", 64L, "application/zip"));
|
||||
when(fileContentStorage.readBlob("blobs/blob-20")).thenReturn(createZipArchive(Map.of(
|
||||
"archive/", "",
|
||||
"archive/nested/", "",
|
||||
"archive/notes.txt", "hello",
|
||||
"archive/nested/todo.txt", "world"
|
||||
)));
|
||||
|
||||
FileService.ZipCompatibleArchive zipArchive = fileService.readZipCompatibleArchive(archive);
|
||||
|
||||
assertThat(zipArchive.commonRootDirectoryName()).isEqualTo("archive");
|
||||
assertThat(zipArchive.entries())
|
||||
.extracting(FileService.ZipCompatibleArchiveEntry::relativePath, FileService.ZipCompatibleArchiveEntry::directory)
|
||||
.containsExactlyInAnyOrder(
|
||||
org.assertj.core.groups.Tuple.tuple("archive", true),
|
||||
org.assertj.core.groups.Tuple.tuple("archive/nested", true),
|
||||
org.assertj.core.groups.Tuple.tuple("archive/notes.txt", false),
|
||||
org.assertj.core.groups.Tuple.tuple("archive/nested/todo.txt", false)
|
||||
);
|
||||
Map<String, String> fileEntries = zipArchive.entries().stream()
|
||||
.filter(entry -> !entry.directory())
|
||||
.collect(java.util.stream.Collectors.toMap(
|
||||
FileService.ZipCompatibleArchiveEntry::relativePath,
|
||||
entry -> new String(entry.content(), StandardCharsets.UTF_8),
|
||||
(left, right) -> left,
|
||||
LinkedHashMap::new
|
||||
));
|
||||
assertThat(fileEntries)
|
||||
.containsEntry("archive/notes.txt", "hello")
|
||||
.containsEntry("archive/nested/todo.txt", "world");
|
||||
verify(fileContentStorage).readBlob("blobs/blob-20");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectZipCompatibleArchiveWithTraversalEntry() throws Exception {
|
||||
User user = createUser(7L);
|
||||
StoredFile archive = createFile(21L, user, "/docs", "extract.zip", createBlob(21L, "blobs/blob-21", 32L, "application/zip"));
|
||||
when(fileContentStorage.readBlob("blobs/blob-21")).thenReturn(createZipArchive(Map.of(
|
||||
"../evil.txt", "oops"
|
||||
)));
|
||||
|
||||
assertThatThrownBy(() -> fileService.readZipCompatibleArchive(archive))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessage("压缩包内容不合法");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDeleteWrittenBlobsWhenBatchExternalImportFails() {
|
||||
User user = createUser(8L);
|
||||
StoredFile docs = createDirectory(300L, user, "/", "docs");
|
||||
when(storedFileRepository.findByUserIdAndPathAndFilename(8L, "/", "docs")).thenReturn(Optional.of(docs));
|
||||
when(storedFileRepository.existsByUserIdAndPathAndFilename(8L, "/docs", "first.txt")).thenReturn(false);
|
||||
when(storedFileRepository.existsByUserIdAndPathAndFilename(8L, "/docs", "second.txt")).thenReturn(false);
|
||||
when(fileBlobRepository.save(any(FileBlob.class))).thenAnswer(invocation -> {
|
||||
FileBlob blob = invocation.getArgument(0);
|
||||
blob.setId(System.nanoTime());
|
||||
return blob;
|
||||
});
|
||||
when(storedFileRepository.save(any(StoredFile.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
StoredFile file = invocation.getArgument(0);
|
||||
if ("second.txt".equals(file.getFilename())) {
|
||||
throw new BusinessException(com.yoyuzh.common.ErrorCode.UNKNOWN, "metadata save failed");
|
||||
}
|
||||
file.setId(400L);
|
||||
return file;
|
||||
});
|
||||
|
||||
assertThatThrownBy(() -> fileService.importExternalFilesAtomically(
|
||||
user,
|
||||
List.of(),
|
||||
List.of(
|
||||
new FileService.ExternalFileImport("/docs", "first.txt", "text/plain", "first".getBytes(StandardCharsets.UTF_8)),
|
||||
new FileService.ExternalFileImport("/docs", "second.txt", "text/plain", "second".getBytes(StandardCharsets.UTF_8))
|
||||
)
|
||||
)).isInstanceOf(BusinessException.class)
|
||||
.hasMessage("metadata save failed");
|
||||
|
||||
var objectKeyCaptor = forClass(String.class);
|
||||
verify(fileContentStorage, times(2)).storeBlob(objectKeyCaptor.capture(), eq("text/plain"), any(byte[].class));
|
||||
List<String> writtenKeys = objectKeyCaptor.getAllValues();
|
||||
assertThat(writtenKeys).hasSize(2);
|
||||
verify(fileContentStorage).deleteBlob(writtenKeys.get(0));
|
||||
verify(fileContentStorage).deleteBlob(writtenKeys.get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCreateShareLinkForOwnedFile() {
|
||||
User user = createUser(7L);
|
||||
@@ -845,4 +992,37 @@ class FileServiceTest {
|
||||
policy.setDefaultPolicy(true);
|
||||
return policy;
|
||||
}
|
||||
|
||||
private Map<String, String> readZipEntries(byte[] archiveBytes) throws Exception {
|
||||
Map<String, String> entries = new LinkedHashMap<>();
|
||||
try (ZipInputStream zipInputStream = new ZipInputStream(
|
||||
new ByteArrayInputStream(archiveBytes), StandardCharsets.UTF_8)) {
|
||||
var entry = zipInputStream.getNextEntry();
|
||||
while (entry != null) {
|
||||
entries.put(entry.getName(), entry.isDirectory() ? "" : new String(zipInputStream.readAllBytes(), StandardCharsets.UTF_8));
|
||||
entry = zipInputStream.getNextEntry();
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
private byte[] createZipArchive(Map<String, String> entries) throws IOException {
|
||||
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, StandardCharsets.UTF_8)) {
|
||||
Set<String> createdEntries = new java.util.LinkedHashSet<>();
|
||||
for (Map.Entry<String, String> entry : entries.entrySet()) {
|
||||
if (!createdEntries.add(entry.getKey())) {
|
||||
continue;
|
||||
}
|
||||
zipOutputStream.putNextEntry(new ZipEntry(entry.getKey()));
|
||||
if (!entry.getKey().endsWith("/")) {
|
||||
zipOutputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
zipOutputStream.closeEntry();
|
||||
}
|
||||
zipOutputStream.finish();
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.share.FileShareLinkRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.core;
|
||||
|
||||
import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.auth.User;
|
||||
@@ -1,10 +1,19 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.events;
|
||||
|
||||
import com.yoyuzh.PortalBackendApplication;
|
||||
import com.yoyuzh.admin.AdminMetricsService;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.core.FileBlobRepository;
|
||||
import com.yoyuzh.files.core.FileEntityRepository;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileEntityRepository;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.files.share.FileShareLinkRepository;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.policy;
|
||||
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -56,7 +56,7 @@ class StoragePolicyServiceTest {
|
||||
|
||||
StoragePolicyCapabilities capabilities = storagePolicyService.readCapabilities(policy);
|
||||
assertThat(capabilities.directUpload()).isTrue();
|
||||
assertThat(capabilities.multipartUpload()).isFalse();
|
||||
assertThat(capabilities.multipartUpload()).isTrue();
|
||||
assertThat(capabilities.signedDownloadUrl()).isTrue();
|
||||
assertThat(capabilities.serverProxyDownload()).isTrue();
|
||||
assertThat(capabilities.requiresCors()).isTrue();
|
||||
@@ -1,7 +1,9 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.search;
|
||||
|
||||
import com.yoyuzh.api.v2.ApiV2Exception;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -11,7 +11,13 @@ import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import software.amazon.awssdk.core.ResponseBytes;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
|
||||
import software.amazon.awssdk.services.s3.model.CompletedPart;
|
||||
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
|
||||
@@ -24,9 +30,12 @@ import software.amazon.awssdk.services.s3.presigner.S3Presigner;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest;
|
||||
|
||||
import java.net.URL;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
@@ -160,6 +169,70 @@ class S3FileContentStorageTest {
|
||||
assertThat(deleteCaptor.getValue().key()).isEqualTo("users/7/docs/old.txt");
|
||||
}
|
||||
|
||||
@Test
|
||||
void createMultipartUploadStartsMultipartInConfiguredBucket() {
|
||||
when(s3Client.createMultipartUpload(any(CreateMultipartUploadRequest.class)))
|
||||
.thenReturn(CreateMultipartUploadResponse.builder().uploadId("upload-123").build());
|
||||
|
||||
String uploadId = storage.createMultipartUpload("blobs/object-1", "video/mp4");
|
||||
|
||||
assertThat(uploadId).isEqualTo("upload-123");
|
||||
ArgumentCaptor<CreateMultipartUploadRequest> requestCaptor = ArgumentCaptor.forClass(CreateMultipartUploadRequest.class);
|
||||
verify(s3Client).createMultipartUpload(requestCaptor.capture());
|
||||
assertThat(requestCaptor.getValue().bucket()).isEqualTo("demo-bucket");
|
||||
assertThat(requestCaptor.getValue().key()).isEqualTo("blobs/object-1");
|
||||
assertThat(requestCaptor.getValue().contentType()).isEqualTo("video/mp4");
|
||||
}
|
||||
|
||||
@Test
|
||||
void prepareMultipartPartUploadSignsUploadPartRequest() throws Exception {
|
||||
PresignedUploadPartRequest presignedRequest = org.mockito.Mockito.mock(PresignedUploadPartRequest.class);
|
||||
when(presignedRequest.url()).thenReturn(new URL("https://upload.example.com/blobs/object-1?partNumber=2"));
|
||||
when(s3Presigner.presignUploadPart(any(UploadPartPresignRequest.class))).thenReturn(presignedRequest);
|
||||
|
||||
PreparedUpload preparedUpload = storage.prepareMultipartPartUpload("blobs/object-1", "upload-123", 2, "video/mp4", 1024L);
|
||||
|
||||
assertThat(preparedUpload.direct()).isTrue();
|
||||
assertThat(preparedUpload.method()).isEqualTo("PUT");
|
||||
assertThat(preparedUpload.uploadUrl()).contains("partNumber=2");
|
||||
ArgumentCaptor<UploadPartPresignRequest> requestCaptor = ArgumentCaptor.forClass(UploadPartPresignRequest.class);
|
||||
verify(s3Presigner).presignUploadPart(requestCaptor.capture());
|
||||
assertThat(requestCaptor.getValue().uploadPartRequest().bucket()).isEqualTo("demo-bucket");
|
||||
assertThat(requestCaptor.getValue().uploadPartRequest().key()).isEqualTo("blobs/object-1");
|
||||
assertThat(requestCaptor.getValue().uploadPartRequest().partNumber()).isEqualTo(2);
|
||||
assertThat(requestCaptor.getValue().uploadPartRequest().uploadId()).isEqualTo("upload-123");
|
||||
}
|
||||
|
||||
@Test
|
||||
void completeMultipartUploadSubmitsSortedCompletedParts() {
|
||||
when(s3Client.completeMultipartUpload(any(CompleteMultipartUploadRequest.class)))
|
||||
.thenReturn(CompleteMultipartUploadResponse.builder().build());
|
||||
|
||||
storage.completeMultipartUpload("blobs/object-1", "upload-123", List.of(
|
||||
new MultipartCompletedPart(2, "etag-2"),
|
||||
new MultipartCompletedPart(1, "etag-1")
|
||||
));
|
||||
|
||||
ArgumentCaptor<CompleteMultipartUploadRequest> requestCaptor = ArgumentCaptor.forClass(CompleteMultipartUploadRequest.class);
|
||||
verify(s3Client).completeMultipartUpload(requestCaptor.capture());
|
||||
assertThat(requestCaptor.getValue().bucket()).isEqualTo("demo-bucket");
|
||||
assertThat(requestCaptor.getValue().key()).isEqualTo("blobs/object-1");
|
||||
assertThat(requestCaptor.getValue().uploadId()).isEqualTo("upload-123");
|
||||
assertThat(requestCaptor.getValue().multipartUpload().parts()).extracting(CompletedPart::partNumber)
|
||||
.containsExactly(1, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void abortMultipartUploadCancelsRemoteMultipartState() {
|
||||
storage.abortMultipartUpload("blobs/object-1", "upload-123");
|
||||
|
||||
ArgumentCaptor<AbortMultipartUploadRequest> requestCaptor = ArgumentCaptor.forClass(AbortMultipartUploadRequest.class);
|
||||
verify(s3Client).abortMultipartUpload(requestCaptor.capture());
|
||||
assertThat(requestCaptor.getValue().bucket()).isEqualTo("demo-bucket");
|
||||
assertThat(requestCaptor.getValue().key()).isEqualTo("blobs/object-1");
|
||||
assertThat(requestCaptor.getValue().uploadId()).isEqualTo("upload-123");
|
||||
}
|
||||
|
||||
@Test
|
||||
void readFileFallsBackToLegacyObjectKeyWhenNeeded() {
|
||||
when(s3Client.headObject(HeadObjectRequest.builder().bucket("demo-bucket").key("users/7/docs/notes.txt").build()))
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.FileMetadataResponse;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BackgroundTaskArchiveHandlerTest {
|
||||
|
||||
@Mock
|
||||
private StoredFileRepository storedFileRepository;
|
||||
@Mock
|
||||
private UserRepository userRepository;
|
||||
@Mock
|
||||
private FileService fileService;
|
||||
|
||||
private ArchiveBackgroundTaskHandler handler;
|
||||
private ArgumentCaptor<byte[]> archiveBytesCaptor;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
handler = new ArchiveBackgroundTaskHandler(
|
||||
storedFileRepository,
|
||||
userRepository,
|
||||
fileService,
|
||||
new ObjectMapper()
|
||||
);
|
||||
archiveBytesCaptor = ArgumentCaptor.forClass(byte[].class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldArchiveDirectoryAndImportZipIntoSameParentPath() throws Exception {
|
||||
User user = createUser(7L);
|
||||
StoredFile directory = createDirectory(11L, user, "/docs", "archive");
|
||||
StoredFile nestedDirectory = createDirectory(12L, user, "/docs/archive", "nested");
|
||||
StoredFile childFile = createFile(13L, user, "/docs/archive", "notes.txt", "text/plain", "blobs/blob-13");
|
||||
StoredFile nestedFile = createFile(14L, user, "/docs/archive/nested", "todo.txt", "text/plain", "blobs/blob-14");
|
||||
FileMetadataResponse importedArchive = new FileMetadataResponse(
|
||||
99L,
|
||||
"archive.zip",
|
||||
"/docs",
|
||||
123L,
|
||||
"application/zip",
|
||||
false,
|
||||
null
|
||||
);
|
||||
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(directory));
|
||||
when(userRepository.findById(7L)).thenReturn(Optional.of(user));
|
||||
when(fileService.summarizeArchiveSource(directory)).thenReturn(new FileService.ArchiveSourceSummary(2, 2));
|
||||
when(fileService.buildArchiveBytes(eq(directory), any())).thenReturn(buildArchiveBytes(Map.of(
|
||||
"archive/", "",
|
||||
"archive/nested/", "",
|
||||
"archive/notes.txt", "hello",
|
||||
"archive/nested/todo.txt", "world"
|
||||
)));
|
||||
when(fileService.importExternalFile(eq(user), eq("/docs"), eq("archive.zip"), eq("application/zip"), anyLong(), any(byte[].class)))
|
||||
.thenReturn(importedArchive);
|
||||
|
||||
BackgroundTaskHandlerResult result = handler.handle(createArchiveTask(11L, 7L));
|
||||
|
||||
verify(fileService).importExternalFile(
|
||||
eq(user),
|
||||
eq("/docs"),
|
||||
eq("archive.zip"),
|
||||
eq("application/zip"),
|
||||
anyLong(),
|
||||
archiveBytesCaptor.capture()
|
||||
);
|
||||
|
||||
Map<String, String> entries = readZipEntries(archiveBytesCaptor.getValue());
|
||||
|
||||
assertThat(entries).containsEntry("archive/", "");
|
||||
assertThat(entries).containsEntry("archive/nested/", "");
|
||||
assertThat(entries).containsEntry("archive/notes.txt", "hello");
|
||||
assertThat(entries).containsEntry("archive/nested/todo.txt", "world");
|
||||
assertThat(result.publicStatePatch()).containsEntry("worker", "archive");
|
||||
assertThat(result.publicStatePatch()).containsEntry("archivedFileId", 99L);
|
||||
assertThat(result.publicStatePatch()).containsEntry("archivedFilename", "archive.zip");
|
||||
assertThat(result.publicStatePatch()).containsEntry("archivedPath", "/docs");
|
||||
assertThat(result.publicStatePatch()).containsEntry("processedFileCount", 2);
|
||||
assertThat(result.publicStatePatch()).containsEntry("totalFileCount", 2);
|
||||
assertThat(result.publicStatePatch()).containsEntry("processedDirectoryCount", 2);
|
||||
assertThat(result.publicStatePatch()).containsEntry("totalDirectoryCount", 2);
|
||||
verify(fileService).buildArchiveBytes(eq(directory), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldArchiveSingleFileIntoZipWithoutLoadingDescendants() throws Exception {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createFile(21L, user, "/docs", "notes.txt", "text/plain", "blobs/blob-21");
|
||||
FileMetadataResponse importedArchive = new FileMetadataResponse(
|
||||
100L,
|
||||
"notes.txt.zip",
|
||||
"/docs",
|
||||
12L,
|
||||
"application/zip",
|
||||
false,
|
||||
null
|
||||
);
|
||||
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(21L, 7L)).thenReturn(Optional.of(file));
|
||||
when(userRepository.findById(7L)).thenReturn(Optional.of(user));
|
||||
when(fileService.summarizeArchiveSource(file)).thenReturn(new FileService.ArchiveSourceSummary(1, 0));
|
||||
when(fileService.buildArchiveBytes(eq(file), any())).thenReturn(buildArchiveBytes(Map.of(
|
||||
"notes.txt", "hello"
|
||||
)));
|
||||
when(fileService.importExternalFile(eq(user), eq("/docs"), eq("notes.txt.zip"), eq("application/zip"), anyLong(), any(byte[].class)))
|
||||
.thenReturn(importedArchive);
|
||||
|
||||
handler.handle(createArchiveTask(21L, 7L));
|
||||
|
||||
verify(storedFileRepository, never()).findByUserIdAndPathEqualsOrDescendant(anyLong(), any());
|
||||
verify(fileService).importExternalFile(
|
||||
eq(user),
|
||||
eq("/docs"),
|
||||
eq("notes.txt.zip"),
|
||||
eq("application/zip"),
|
||||
anyLong(),
|
||||
archiveBytesCaptor.capture()
|
||||
);
|
||||
|
||||
Map<String, String> entries = readZipEntries(archiveBytesCaptor.getValue());
|
||||
assertThat(entries).containsEntry("notes.txt", "hello");
|
||||
}
|
||||
|
||||
private BackgroundTask createArchiveTask(Long fileId, Long userId) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setId(301L);
|
||||
task.setType(BackgroundTaskType.ARCHIVE);
|
||||
task.setStatus(BackgroundTaskStatus.RUNNING);
|
||||
task.setUserId(userId);
|
||||
task.setPublicStateJson("{\"fileId\":" + fileId + "}");
|
||||
task.setPrivateStateJson("{\"fileId\":" + fileId + ",\"taskType\":\"ARCHIVE\",\"outputPath\":\"/docs\",\"outputFilename\":\""
|
||||
+ (fileId.equals(21L) ? "notes.txt.zip" : "archive.zip") + "\"}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private User createUser(Long id) {
|
||||
User user = new User();
|
||||
user.setId(id);
|
||||
user.setUsername("alice");
|
||||
return user;
|
||||
}
|
||||
|
||||
private StoredFile createDirectory(Long id, User user, String path, String filename) {
|
||||
StoredFile file = new StoredFile();
|
||||
file.setId(id);
|
||||
file.setUser(user);
|
||||
file.setPath(path);
|
||||
file.setFilename(filename);
|
||||
file.setDirectory(true);
|
||||
file.setSize(0L);
|
||||
return file;
|
||||
}
|
||||
|
||||
private StoredFile createFile(Long id,
|
||||
User user,
|
||||
String path,
|
||||
String filename,
|
||||
String contentType,
|
||||
String objectKey) {
|
||||
StoredFile file = new StoredFile();
|
||||
file.setId(id);
|
||||
file.setUser(user);
|
||||
file.setPath(path);
|
||||
file.setFilename(filename);
|
||||
file.setDirectory(false);
|
||||
file.setContentType(contentType);
|
||||
file.setSize(5L);
|
||||
FileBlob blob = new FileBlob();
|
||||
blob.setId(id + 1000);
|
||||
blob.setObjectKey(objectKey);
|
||||
blob.setContentType(contentType);
|
||||
blob.setSize(5L);
|
||||
file.setBlob(blob);
|
||||
return file;
|
||||
}
|
||||
|
||||
private Map<String, String> readZipEntries(byte[] archiveBytes) throws Exception {
|
||||
Map<String, String> entries = new LinkedHashMap<>();
|
||||
try (ZipInputStream zipInputStream = new ZipInputStream(
|
||||
new ByteArrayInputStream(archiveBytes), StandardCharsets.UTF_8)) {
|
||||
var entry = zipInputStream.getNextEntry();
|
||||
while (entry != null) {
|
||||
entries.put(entry.getName(), entry.isDirectory() ? "" : new String(zipInputStream.readAllBytes(), StandardCharsets.UTF_8));
|
||||
entry = zipInputStream.getNextEntry();
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
private byte[] buildArchiveBytes(Map<String, String> entries) throws Exception {
|
||||
java.io.ByteArrayOutputStream outputStream = new java.io.ByteArrayOutputStream();
|
||||
try (java.util.zip.ZipOutputStream zipOutputStream = new java.util.zip.ZipOutputStream(outputStream, StandardCharsets.UTF_8)) {
|
||||
for (Map.Entry<String, String> entry : entries.entrySet()) {
|
||||
zipOutputStream.putNextEntry(new java.util.zip.ZipEntry(entry.getKey()));
|
||||
if (!entry.getKey().endsWith("/")) {
|
||||
zipOutputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
zipOutputStream.closeEntry();
|
||||
}
|
||||
}
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,614 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.api.v2.ApiV2Exception;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BackgroundTaskServiceTest {
|
||||
|
||||
@Mock
|
||||
private BackgroundTaskRepository backgroundTaskRepository;
|
||||
|
||||
@Mock
|
||||
private StoredFileRepository storedFileRepository;
|
||||
|
||||
private BackgroundTaskService backgroundTaskService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
backgroundTaskService = new BackgroundTaskService(backgroundTaskRepository, storedFileRepository, new ObjectMapper());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationForForeignFile() {
|
||||
User user = createUser(7L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(99L, 7L)).thenReturn(Optional.empty());
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
99L,
|
||||
"/docs/foreign.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("file not found");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationForDeletedFile() {
|
||||
User user = createUser(7L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(100L, 7L)).thenReturn(Optional.empty());
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
100L,
|
||||
"/docs/deleted.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("file not found");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectTaskCreationWhenRequestedPathDoesNotMatchFile() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(11L, user, "/docs", "real.txt", false, "text/plain", 3L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(file));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
11L,
|
||||
"/docs/fake.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("task path does not match file path");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectExtractTaskForDirectory() {
|
||||
User user = createUser(7L);
|
||||
StoredFile directory = createStoredFile(12L, user, "/", "bundle", true, null, 0L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(12L, 7L)).thenReturn(Optional.of(directory));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.EXTRACT,
|
||||
12L,
|
||||
"/bundle",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("task target type is not supported");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectExtractTaskForNonZipCompatibleArchive() {
|
||||
User user = createUser(7L);
|
||||
StoredFile archive = createStoredFile(17L, user, "/docs", "backup.7z", false, "application/x-7z-compressed", 64L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(17L, 7L)).thenReturn(Optional.of(archive));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.EXTRACT,
|
||||
17L,
|
||||
"/docs/backup.7z",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("extract task only supports zip-compatible archives");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectMediaMetadataTaskForNonMediaFile() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(13L, user, "/docs", "notes.txt", false, "text/plain", 9L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(13L, 7L)).thenReturn(Optional.of(file));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.MEDIA_META,
|
||||
13L,
|
||||
"/docs/notes.txt",
|
||||
null
|
||||
)).isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("media metadata task only supports media files");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCreateTaskStateFromServerFilePath() {
|
||||
User user = createUser(7L);
|
||||
StoredFile file = createStoredFile(14L, user, "/docs", "photo.png", false, "image/png", 15L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(14L, 7L)).thenReturn(Optional.of(file));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask task = backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.MEDIA_META,
|
||||
14L,
|
||||
"/docs/photo.png",
|
||||
"media-1"
|
||||
);
|
||||
|
||||
assertThat(task.getPublicStateJson()).contains("\"fileId\":14");
|
||||
assertThat(task.getPublicStateJson()).contains("\"path\":\"/docs/photo.png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"filename\":\"photo.png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"directory\":false");
|
||||
assertThat(task.getPublicStateJson()).contains("\"contentType\":\"image/png\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"size\":15");
|
||||
assertThat(task.getPublicStateJson()).contains("\"phase\":\"queued\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"attemptCount\":0");
|
||||
assertThat(task.getPublicStateJson()).contains("\"maxAttempts\":2");
|
||||
assertThat(task.getPrivateStateJson()).contains("\"taskType\":\"MEDIA_META\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCreateArchiveTaskStateWithDerivedOutputTarget() {
|
||||
User user = createUser(7L);
|
||||
StoredFile directory = createStoredFile(15L, user, "/docs", "archive", true, null, 0L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(15L, 7L)).thenReturn(Optional.of(directory));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask task = backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.ARCHIVE,
|
||||
15L,
|
||||
"/docs/archive",
|
||||
"archive-1"
|
||||
);
|
||||
|
||||
assertThat(task.getPublicStateJson()).contains("\"outputPath\":\"/docs\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"outputFilename\":\"archive.zip\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"maxAttempts\":4");
|
||||
assertThat(task.getPrivateStateJson()).contains("\"outputPath\":\"/docs\"");
|
||||
assertThat(task.getPrivateStateJson()).contains("\"outputFilename\":\"archive.zip\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCreateExtractTaskStateWithDerivedOutputTarget() {
|
||||
User user = createUser(7L);
|
||||
StoredFile archive = createStoredFile(16L, user, "/docs", "extract.zip", false, "application/zip", 32L);
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(16L, 7L)).thenReturn(Optional.of(archive));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask task = backgroundTaskService.createQueuedFileTask(
|
||||
user,
|
||||
BackgroundTaskType.EXTRACT,
|
||||
16L,
|
||||
"/docs/extract.zip",
|
||||
"extract-1"
|
||||
);
|
||||
|
||||
assertThat(task.getPublicStateJson()).contains("\"outputPath\":\"/docs\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"outputDirectoryName\":\"extract\"");
|
||||
assertThat(task.getPublicStateJson()).contains("\"maxAttempts\":3");
|
||||
assertThat(task.getPrivateStateJson()).contains("\"outputPath\":\"/docs\"");
|
||||
assertThat(task.getPrivateStateJson()).contains("\"outputDirectoryName\":\"extract\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldClaimQueuedTaskOnlyWhenRepositoryTransitionSucceeds() {
|
||||
BackgroundTask task = createTask(1L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskRepository.claimQueuedTask(
|
||||
eq(1L),
|
||||
eq(BackgroundTaskStatus.QUEUED),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq("worker-a"),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(1L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
Optional<BackgroundTask> result = backgroundTaskService.claimQueuedTask(1L, "worker-a", 120L);
|
||||
|
||||
assertThat(result).containsSame(task);
|
||||
assertThat(result.orElseThrow().getLeaseOwner()).isEqualTo("worker-a");
|
||||
assertThat(result.orElseThrow().getLeaseExpiresAt()).isNotNull();
|
||||
assertThat(result.orElseThrow().getHeartbeatAt()).isNotNull();
|
||||
assertThat(result.orElseThrow().getPublicStateJson()).contains("\"phase\":\"running\"");
|
||||
assertThat(result.orElseThrow().getPublicStateJson()).contains("\"attemptCount\":1");
|
||||
assertThat(result.orElseThrow().getPublicStateJson()).contains("\"maxAttempts\":4");
|
||||
assertThat(result.orElseThrow().getPublicStateJson()).contains("\"workerOwner\":\"worker-a\"");
|
||||
assertThat(result.orElseThrow().getPublicStateJson()).contains("\"heartbeatAt\":");
|
||||
assertThat(result.orElseThrow().getPublicStateJson()).contains("\"leaseExpiresAt\":");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldNotClaimTaskWhenRepositoryTransitionWasSkipped() {
|
||||
when(backgroundTaskRepository.claimQueuedTask(
|
||||
eq(2L),
|
||||
eq(BackgroundTaskStatus.QUEUED),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq("worker-a"),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(0);
|
||||
|
||||
Optional<BackgroundTask> result = backgroundTaskService.claimQueuedTask(2L, "worker-a", 120L);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldCompleteRunningWorkerTaskAndMergePublicState() {
|
||||
BackgroundTask task = createTask(3L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
|
||||
task.setLeaseOwner("worker-a");
|
||||
task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120));
|
||||
task.setHeartbeatAt(java.time.LocalDateTime.now());
|
||||
task.setPublicStateJson("{\"fileId\":11,\"phase\":\"archiving\"}");
|
||||
when(backgroundTaskRepository.refreshRunningTaskLease(
|
||||
eq(3L),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq("worker-a"),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(3L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskCompleted(3L, "worker-a", Map.of("worker", "noop"), 120L);
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.COMPLETED);
|
||||
assertThat(result.getFinishedAt()).isNotNull();
|
||||
assertThat(result.getErrorMessage()).isNull();
|
||||
assertThat(result.getLeaseOwner()).isNull();
|
||||
assertThat(result.getLeaseExpiresAt()).isNull();
|
||||
assertThat(result.getHeartbeatAt()).isNull();
|
||||
assertThat(result.getPublicStateJson()).contains("\"fileId\":11");
|
||||
assertThat(result.getPublicStateJson()).contains("\"worker\":\"noop\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"phase\":\"completed\"");
|
||||
assertThat(result.getPublicStateJson()).doesNotContain("workerOwner");
|
||||
assertThat(result.getPublicStateJson()).doesNotContain("leaseExpiresAt");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldMergeWorkerProgressStateWhileTaskIsRunning() {
|
||||
BackgroundTask task = createTask(7L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING);
|
||||
task.setLeaseOwner("worker-a");
|
||||
task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120));
|
||||
task.setHeartbeatAt(java.time.LocalDateTime.now());
|
||||
task.setPublicStateJson("{\"fileId\":11,\"phase\":\"running\"}");
|
||||
when(backgroundTaskRepository.refreshRunningTaskLease(
|
||||
eq(7L),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq("worker-a"),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(7L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskProgress(
|
||||
7L,
|
||||
"worker-a",
|
||||
Map.of("phase", "extracting", "progressPercent", 50),
|
||||
120L
|
||||
);
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.RUNNING);
|
||||
assertThat(result.getPublicStateJson()).contains("\"phase\":\"extracting\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"progressPercent\":50");
|
||||
assertThat(result.getPublicStateJson()).contains("\"workerOwner\":\"worker-a\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"heartbeatAt\":");
|
||||
assertThat(result.getPublicStateJson()).contains("\"leaseExpiresAt\":");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRecordTerminalWorkerFailureMessageWhenFailureIsNotRetryable() {
|
||||
BackgroundTask task = createTask(4L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING);
|
||||
task.setLeaseOwner("worker-a");
|
||||
task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120));
|
||||
task.setHeartbeatAt(java.time.LocalDateTime.now());
|
||||
task.setPublicStateJson("{\"fileId\":11,\"phase\":\"extracting\"}");
|
||||
when(backgroundTaskRepository.refreshRunningTaskLease(
|
||||
eq(4L),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq("worker-a"),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(4L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskFailed(
|
||||
4L,
|
||||
"worker-a",
|
||||
"media parser unavailable",
|
||||
BackgroundTaskFailureCategory.DATA_STATE,
|
||||
120L
|
||||
);
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED);
|
||||
assertThat(result.getFinishedAt()).isNotNull();
|
||||
assertThat(result.getErrorMessage()).isEqualTo("media parser unavailable");
|
||||
assertThat(result.getLeaseOwner()).isNull();
|
||||
assertThat(result.getLeaseExpiresAt()).isNull();
|
||||
assertThat(result.getHeartbeatAt()).isNull();
|
||||
assertThat(result.getPublicStateJson()).contains("\"phase\":\"failed\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"attemptCount\":1");
|
||||
assertThat(result.getPublicStateJson()).contains("\"maxAttempts\":3");
|
||||
assertThat(result.getPublicStateJson()).contains("\"failureCategory\":\"DATA_STATE\"");
|
||||
assertThat(result.getPublicStateJson()).doesNotContain("retryScheduled");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRequeueRetryableWorkerFailureWithBackoff() {
|
||||
BackgroundTask task = createTask(14L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
|
||||
task.setLeaseOwner("worker-a");
|
||||
task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120));
|
||||
task.setHeartbeatAt(java.time.LocalDateTime.now());
|
||||
task.setPublicStateJson("{\"fileId\":11,\"phase\":\"extracting\"}");
|
||||
when(backgroundTaskRepository.refreshRunningTaskLease(
|
||||
eq(14L),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq("worker-a"),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(14L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskFailed(
|
||||
14L,
|
||||
"worker-a",
|
||||
"storage timeout",
|
||||
BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE,
|
||||
120L
|
||||
);
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED);
|
||||
assertThat(result.getFinishedAt()).isNull();
|
||||
assertThat(result.getErrorMessage()).isNull();
|
||||
assertThat(result.getNextRunAt()).isNotNull();
|
||||
assertThat(result.getLeaseOwner()).isNull();
|
||||
assertThat(result.getLeaseExpiresAt()).isNull();
|
||||
assertThat(result.getHeartbeatAt()).isNull();
|
||||
assertThat(result.getPublicStateJson()).contains("\"phase\":\"queued\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"retryScheduled\":true");
|
||||
assertThat(result.getPublicStateJson()).contains("\"retryDelaySeconds\":30");
|
||||
assertThat(result.getPublicStateJson()).contains("\"failureCategory\":\"TRANSIENT_INFRASTRUCTURE\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"lastFailureMessage\":\"storage timeout\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUseLongerBackoffForRateLimitedFailures() {
|
||||
BackgroundTask task = createTask(18L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
|
||||
task.setLeaseOwner("worker-a");
|
||||
task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120));
|
||||
task.setHeartbeatAt(java.time.LocalDateTime.now());
|
||||
task.setPublicStateJson("{\"fileId\":11,\"phase\":\"archiving\"}");
|
||||
when(backgroundTaskRepository.refreshRunningTaskLease(
|
||||
eq(18L),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq("worker-a"),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(18L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskFailed(
|
||||
18L,
|
||||
"worker-a",
|
||||
"429 too many requests",
|
||||
BackgroundTaskFailureCategory.RATE_LIMITED,
|
||||
120L
|
||||
);
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED);
|
||||
assertThat(result.getPublicStateJson()).contains("\"retryDelaySeconds\":120");
|
||||
assertThat(result.getPublicStateJson()).contains("\"failureCategory\":\"RATE_LIMITED\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFailTerminallyWhenRetryableFailureExhaustsAttempts() {
|
||||
BackgroundTask task = createTask(15L, BackgroundTaskType.MEDIA_META, BackgroundTaskStatus.RUNNING);
|
||||
task.setAttemptCount(2);
|
||||
task.setMaxAttempts(2);
|
||||
task.setLeaseOwner("worker-a");
|
||||
task.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(120));
|
||||
task.setHeartbeatAt(java.time.LocalDateTime.now());
|
||||
task.setPublicStateJson("{\"fileId\":11,\"phase\":\"extracting-metadata\",\"attemptCount\":2,\"maxAttempts\":2}");
|
||||
when(backgroundTaskRepository.refreshRunningTaskLease(
|
||||
eq(15L),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq("worker-a"),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(15L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.markWorkerTaskFailed(
|
||||
15L,
|
||||
"worker-a",
|
||||
"storage timeout",
|
||||
BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE,
|
||||
120L
|
||||
);
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.FAILED);
|
||||
assertThat(result.getFinishedAt()).isNotNull();
|
||||
assertThat(result.getErrorMessage()).isEqualTo("storage timeout");
|
||||
assertThat(result.getPublicStateJson()).contains("\"phase\":\"failed\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"failureCategory\":\"TRANSIENT_INFRASTRUCTURE\"");
|
||||
assertThat(result.getPublicStateJson()).doesNotContain("retryScheduled");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRetryFailedTaskAndResetPublicState() {
|
||||
User user = createUser(7L);
|
||||
BackgroundTask task = createTask(8L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.FAILED);
|
||||
task.setPublicStateJson("""
|
||||
{"fileId":11,"path":"/docs/extract.zip","phase":"failed","worker":"extract","processedFileCount":1,"totalFileCount":2}
|
||||
""");
|
||||
task.setPrivateStateJson("""
|
||||
{"fileId":11,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"}
|
||||
""");
|
||||
task.setFinishedAt(java.time.LocalDateTime.now());
|
||||
task.setErrorMessage("extract task only supports zip-compatible archives");
|
||||
when(backgroundTaskRepository.findByIdAndUserId(8L, 7L)).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
BackgroundTask result = backgroundTaskService.retryOwnedTask(user, 8L);
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED);
|
||||
assertThat(result.getFinishedAt()).isNull();
|
||||
assertThat(result.getErrorMessage()).isNull();
|
||||
assertThat(result.getPublicStateJson()).contains("\"phase\":\"queued\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"outputPath\":\"/docs\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"outputDirectoryName\":\"extract\"");
|
||||
assertThat(result.getPublicStateJson()).contains("\"attemptCount\":0");
|
||||
assertThat(result.getPublicStateJson()).contains("\"maxAttempts\":3");
|
||||
assertThat(result.getPublicStateJson()).doesNotContain("taskType");
|
||||
assertThat(result.getPublicStateJson()).doesNotContain("worker");
|
||||
assertThat(result.getPublicStateJson()).doesNotContain("processedFileCount");
|
||||
assertThat(result.getPublicStateJson()).doesNotContain("totalFileCount");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectRetryForNonFailedTask() {
|
||||
User user = createUser(7L);
|
||||
BackgroundTask task = createTask(9L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.COMPLETED);
|
||||
when(backgroundTaskRepository.findByIdAndUserId(9L, 7L)).thenReturn(Optional.of(task));
|
||||
|
||||
assertThatThrownBy(() -> backgroundTaskService.retryOwnedTask(user, 9L))
|
||||
.isInstanceOf(ApiV2Exception.class)
|
||||
.hasMessage("only failed tasks can be retried");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRequeueOnlyExpiredRunningTasksOnStartup() {
|
||||
BackgroundTask expired = createTask(10L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING);
|
||||
expired.setLeaseOwner("worker-stale");
|
||||
expired.setLeaseExpiresAt(java.time.LocalDateTime.now().minusSeconds(5));
|
||||
expired.setHeartbeatAt(java.time.LocalDateTime.now().minusSeconds(10));
|
||||
expired.setPublicStateJson("""
|
||||
{"fileId":11,"path":"/docs/extract.zip","phase":"extracting","worker":"extract","workerOwner":"worker-stale"}
|
||||
""");
|
||||
expired.setPrivateStateJson("""
|
||||
{"fileId":11,"path":"/docs/extract.zip","taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"extract"}
|
||||
""");
|
||||
expired.setFinishedAt(java.time.LocalDateTime.now());
|
||||
expired.setErrorMessage("partial failure");
|
||||
BackgroundTask fresh = createTask(11L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING);
|
||||
fresh.setLeaseOwner("worker-live");
|
||||
fresh.setLeaseExpiresAt(java.time.LocalDateTime.now().plusSeconds(300));
|
||||
fresh.setHeartbeatAt(java.time.LocalDateTime.now());
|
||||
when(backgroundTaskRepository.findExpiredRunningTaskIds(eq(BackgroundTaskStatus.RUNNING), any(), any()))
|
||||
.thenReturn(List.of(10L));
|
||||
when(backgroundTaskRepository.requeueExpiredRunningTask(
|
||||
eq(10L),
|
||||
eq(BackgroundTaskStatus.RUNNING),
|
||||
eq(BackgroundTaskStatus.QUEUED),
|
||||
any(),
|
||||
any()
|
||||
)).thenReturn(1);
|
||||
when(backgroundTaskRepository.findById(10L)).thenReturn(Optional.of(expired));
|
||||
when(backgroundTaskRepository.save(any(BackgroundTask.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
int recovered = backgroundTaskService.requeueExpiredRunningTasks();
|
||||
|
||||
assertThat(recovered).isEqualTo(1);
|
||||
assertThat(expired.getStatus()).isEqualTo(BackgroundTaskStatus.QUEUED);
|
||||
assertThat(expired.getFinishedAt()).isNull();
|
||||
assertThat(expired.getErrorMessage()).isNull();
|
||||
assertThat(expired.getLeaseOwner()).isNull();
|
||||
assertThat(expired.getLeaseExpiresAt()).isNull();
|
||||
assertThat(expired.getHeartbeatAt()).isNull();
|
||||
assertThat(expired.getPublicStateJson()).contains("\"phase\":\"queued\"");
|
||||
assertThat(expired.getPublicStateJson()).contains("\"attemptCount\":1");
|
||||
assertThat(expired.getPublicStateJson()).contains("\"maxAttempts\":3");
|
||||
assertThat(expired.getPublicStateJson()).contains("\"outputPath\":\"/docs\"");
|
||||
assertThat(expired.getPublicStateJson()).contains("\"outputDirectoryName\":\"extract\"");
|
||||
assertThat(expired.getPublicStateJson()).doesNotContain("worker");
|
||||
assertThat(expired.getPublicStateJson()).doesNotContain("taskType");
|
||||
assertThat(expired.getPublicStateJson()).doesNotContain("workerOwner");
|
||||
assertThat(fresh.getStatus()).isEqualTo(BackgroundTaskStatus.RUNNING);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFindQueuedTaskIdsInCreatedOrderLimit() {
|
||||
when(backgroundTaskRepository.findReadyTaskIdsByStatusOrder(eq(BackgroundTaskStatus.QUEUED), any(), any()))
|
||||
.thenReturn(List.of(5L, 6L));
|
||||
|
||||
List<Long> result = backgroundTaskService.findQueuedTaskIds(2);
|
||||
|
||||
assertThat(result).containsExactly(5L, 6L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnEmptyTaskIdsWhenLimitIsNonPositive() {
|
||||
List<Long> result = backgroundTaskService.findQueuedTaskIds(0);
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
private BackgroundTask createTask(Long id, BackgroundTaskType type, BackgroundTaskStatus status) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setId(id);
|
||||
task.setType(type);
|
||||
task.setStatus(status);
|
||||
task.setUserId(7L);
|
||||
task.setAttemptCount(status == BackgroundTaskStatus.RUNNING ? 1 : 0);
|
||||
task.setMaxAttempts(switch (type) {
|
||||
case ARCHIVE -> 4;
|
||||
case EXTRACT -> 3;
|
||||
case MEDIA_META -> 2;
|
||||
default -> 1;
|
||||
});
|
||||
task.setPublicStateJson("{}");
|
||||
task.setPrivateStateJson("{}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private User createUser(Long id) {
|
||||
User user = new User();
|
||||
user.setId(id);
|
||||
user.setUsername("alice");
|
||||
return user;
|
||||
}
|
||||
|
||||
private StoredFile createStoredFile(Long id,
|
||||
User user,
|
||||
String path,
|
||||
String filename,
|
||||
boolean directory,
|
||||
String contentType,
|
||||
Long size) {
|
||||
StoredFile file = new StoredFile();
|
||||
file.setId(id);
|
||||
file.setUser(user);
|
||||
file.setPath(path);
|
||||
file.setFilename(filename);
|
||||
file.setDirectory(directory);
|
||||
file.setContentType(contentType);
|
||||
file.setSize(size);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BackgroundTaskWorkerTest {
|
||||
|
||||
@Mock
|
||||
private BackgroundTaskService backgroundTaskService;
|
||||
@Mock
|
||||
private BackgroundTaskHandler backgroundTaskHandler;
|
||||
|
||||
private BackgroundTaskWorker backgroundTaskWorker;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
backgroundTaskWorker = new BackgroundTaskWorker(backgroundTaskService, List.of(backgroundTaskHandler));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldClaimAndCompleteQueuedTaskThroughNoopHandler() {
|
||||
BackgroundTask task = createTask(1L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L));
|
||||
when(backgroundTaskService.claimQueuedTask(eq(1L), anyString(), anyLong())).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskHandler.supports(BackgroundTaskType.ARCHIVE)).thenReturn(true);
|
||||
when(backgroundTaskHandler.handle(eq(task), any(BackgroundTaskProgressReporter.class)))
|
||||
.thenReturn(new BackgroundTaskHandlerResult(Map.of("worker", "noop")));
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
verify(backgroundTaskService).markWorkerTaskProgress(eq(1L), anyString(), eq(Map.of("phase", "archiving")), anyLong());
|
||||
verify(backgroundTaskHandler).handle(eq(task), any(BackgroundTaskProgressReporter.class));
|
||||
verify(backgroundTaskService).markWorkerTaskCompleted(eq(1L), anyString(), eq(Map.of("worker", "noop")), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldSkipTaskThatWasNotClaimed() {
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(1L));
|
||||
when(backgroundTaskService.claimQueuedTask(eq(1L), anyString(), anyLong())).thenReturn(Optional.empty());
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isZero();
|
||||
verify(backgroundTaskHandler, never()).handle(any(BackgroundTask.class), any(BackgroundTaskProgressReporter.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldMarkTaskFailedWhenHandlerThrows() {
|
||||
BackgroundTask task = createTask(2L, BackgroundTaskType.MEDIA_META, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(2L));
|
||||
when(backgroundTaskService.claimQueuedTask(eq(2L), anyString(), anyLong())).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskHandler.supports(BackgroundTaskType.MEDIA_META)).thenReturn(true);
|
||||
when(backgroundTaskHandler.handle(eq(task), any(BackgroundTaskProgressReporter.class)))
|
||||
.thenThrow(new IllegalStateException("media parser unavailable"));
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
verify(backgroundTaskService).markWorkerTaskProgress(eq(2L), anyString(), eq(Map.of("phase", "extracting-metadata")), anyLong());
|
||||
verify(backgroundTaskService).markWorkerTaskFailed(
|
||||
eq(2L),
|
||||
anyString(),
|
||||
eq("media parser unavailable"),
|
||||
eq(BackgroundTaskFailureCategory.DATA_STATE),
|
||||
anyLong()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldAutoRetryUnexpectedWorkerFailure() {
|
||||
BackgroundTask task = createTask(3L, BackgroundTaskType.ARCHIVE, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(3L));
|
||||
when(backgroundTaskService.claimQueuedTask(eq(3L), anyString(), anyLong())).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskHandler.supports(BackgroundTaskType.ARCHIVE)).thenReturn(true);
|
||||
when(backgroundTaskHandler.handle(eq(task), any(BackgroundTaskProgressReporter.class)))
|
||||
.thenThrow(new RuntimeException("storage timeout"));
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
verify(backgroundTaskService).markWorkerTaskFailed(
|
||||
eq(3L),
|
||||
anyString(),
|
||||
eq("storage timeout"),
|
||||
eq(BackgroundTaskFailureCategory.TRANSIENT_INFRASTRUCTURE),
|
||||
anyLong()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldClassifyRateLimitedFailureSeparately() {
|
||||
BackgroundTask task = createTask(4L, BackgroundTaskType.EXTRACT, BackgroundTaskStatus.RUNNING);
|
||||
when(backgroundTaskService.findQueuedTaskIds(5)).thenReturn(List.of(4L));
|
||||
when(backgroundTaskService.claimQueuedTask(eq(4L), anyString(), anyLong())).thenReturn(Optional.of(task));
|
||||
when(backgroundTaskHandler.supports(BackgroundTaskType.EXTRACT)).thenReturn(true);
|
||||
when(backgroundTaskHandler.handle(eq(task), any(BackgroundTaskProgressReporter.class)))
|
||||
.thenThrow(new RuntimeException("429 Too Many Requests"));
|
||||
|
||||
int processedCount = backgroundTaskWorker.processQueuedTasks(5);
|
||||
|
||||
assertThat(processedCount).isEqualTo(1);
|
||||
verify(backgroundTaskService).markWorkerTaskFailed(
|
||||
eq(4L),
|
||||
anyString(),
|
||||
eq("429 Too Many Requests"),
|
||||
eq(BackgroundTaskFailureCategory.RATE_LIMITED),
|
||||
anyLong()
|
||||
);
|
||||
}
|
||||
|
||||
private BackgroundTask createTask(Long id, BackgroundTaskType type, BackgroundTaskStatus status) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setId(id);
|
||||
task.setType(type);
|
||||
task.setStatus(status);
|
||||
task.setUserId(7L);
|
||||
task.setPublicStateJson("{}");
|
||||
task.setPrivateStateJson("{}");
|
||||
return task;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.auth.UserRepository;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.common.ErrorCode;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ExtractBackgroundTaskHandlerTest {
|
||||
|
||||
@Mock
|
||||
private StoredFileRepository storedFileRepository;
|
||||
@Mock
|
||||
private UserRepository userRepository;
|
||||
@Mock
|
||||
private FileService fileService;
|
||||
|
||||
private ExtractBackgroundTaskHandler handler;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
handler = new ExtractBackgroundTaskHandler(
|
||||
storedFileRepository,
|
||||
userRepository,
|
||||
fileService,
|
||||
new ObjectMapper()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldExtractArchivedDirectoryIntoSiblingFolder() throws Exception {
|
||||
User user = createUser(7L);
|
||||
StoredFile archive = createArchiveFile(11L, user, "/docs", "archive.zip", "application/zip", "blobs/archive.zip");
|
||||
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(11L, 7L)).thenReturn(Optional.of(archive));
|
||||
when(userRepository.findById(7L)).thenReturn(Optional.of(user));
|
||||
when(fileService.readZipCompatibleArchive(archive)).thenReturn(new FileService.ZipCompatibleArchive(
|
||||
java.util.List.of(
|
||||
new FileService.ZipCompatibleArchiveEntry("archive", true, new byte[0]),
|
||||
new FileService.ZipCompatibleArchiveEntry("archive/nested", true, new byte[0]),
|
||||
new FileService.ZipCompatibleArchiveEntry("archive/notes.txt", false, "hello".getBytes(StandardCharsets.UTF_8)),
|
||||
new FileService.ZipCompatibleArchiveEntry("archive/nested/todo.txt", false, "world".getBytes(StandardCharsets.UTF_8))
|
||||
),
|
||||
"archive"
|
||||
));
|
||||
|
||||
BackgroundTaskHandlerResult result = handler.handle(createExtractTask(11L, 7L, "archive"));
|
||||
|
||||
verify(fileService).importExternalFilesAtomically(
|
||||
eq(user),
|
||||
eq(java.util.List.of("/docs/archive", "/docs/archive/nested")),
|
||||
argThat(files -> files.size() == 2
|
||||
&& files.stream().anyMatch(file -> "/docs/archive".equals(file.path())
|
||||
&& "notes.txt".equals(file.filename())
|
||||
&& "text/plain".equals(file.contentType())
|
||||
&& java.util.Arrays.equals("hello".getBytes(StandardCharsets.UTF_8), file.content()))
|
||||
&& files.stream().anyMatch(file -> "/docs/archive/nested".equals(file.path())
|
||||
&& "todo.txt".equals(file.filename())
|
||||
&& "text/plain".equals(file.contentType())
|
||||
&& java.util.Arrays.equals("world".getBytes(StandardCharsets.UTF_8), file.content()))),
|
||||
any()
|
||||
);
|
||||
assertThat(result.publicStatePatch()).containsEntry("worker", "extract");
|
||||
assertThat(result.publicStatePatch()).containsEntry("extractedPath", "/docs/archive");
|
||||
assertThat(result.publicStatePatch()).containsEntry("extractedFileCount", 2);
|
||||
assertThat(result.publicStatePatch()).containsEntry("extractedDirectoryCount", 2);
|
||||
assertThat(result.publicStatePatch()).containsEntry("processedFileCount", 2);
|
||||
assertThat(result.publicStatePatch()).containsEntry("totalFileCount", 2);
|
||||
assertThat(result.publicStatePatch()).containsEntry("processedDirectoryCount", 2);
|
||||
assertThat(result.publicStatePatch()).containsEntry("totalDirectoryCount", 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldExtractSingleArchivedFileBackIntoParentPath() throws Exception {
|
||||
User user = createUser(7L);
|
||||
StoredFile archive = createArchiveFile(21L, user, "/docs", "notes.txt.zip", "application/zip", "blobs/notes.txt.zip");
|
||||
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(21L, 7L)).thenReturn(Optional.of(archive));
|
||||
when(userRepository.findById(7L)).thenReturn(Optional.of(user));
|
||||
when(fileService.readZipCompatibleArchive(archive)).thenReturn(new FileService.ZipCompatibleArchive(
|
||||
java.util.List.of(
|
||||
new FileService.ZipCompatibleArchiveEntry("notes.txt", false, "hello".getBytes(StandardCharsets.UTF_8))
|
||||
),
|
||||
null
|
||||
));
|
||||
|
||||
BackgroundTaskHandlerResult result = handler.handle(createExtractTask(21L, 7L, "notes.txt"));
|
||||
|
||||
verify(fileService).importExternalFilesAtomically(
|
||||
eq(user),
|
||||
eq(java.util.List.of()),
|
||||
argThat(files -> files.size() == 1
|
||||
&& "/docs".equals(files.get(0).path())
|
||||
&& "notes.txt".equals(files.get(0).filename())
|
||||
&& "text/plain".equals(files.get(0).contentType())
|
||||
&& java.util.Arrays.equals("hello".getBytes(StandardCharsets.UTF_8), files.get(0).content())),
|
||||
any()
|
||||
);
|
||||
assertThat(result.publicStatePatch()).containsEntry("worker", "extract");
|
||||
assertThat(result.publicStatePatch()).containsEntry("extractedPath", "/docs");
|
||||
assertThat(result.publicStatePatch()).containsEntry("extractedFileCount", 1);
|
||||
assertThat(result.publicStatePatch()).containsEntry("extractedDirectoryCount", 0);
|
||||
assertThat(result.publicStatePatch()).containsEntry("processedFileCount", 1);
|
||||
assertThat(result.publicStatePatch()).containsEntry("totalFileCount", 1);
|
||||
assertThat(result.publicStatePatch()).containsEntry("processedDirectoryCount", 0);
|
||||
assertThat(result.publicStatePatch()).containsEntry("totalDirectoryCount", 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRejectNonZipCompatibleArchiveContent() {
|
||||
User user = createUser(7L);
|
||||
StoredFile archive = createArchiveFile(31L, user, "/docs", "backup.7z", "application/x-7z-compressed", "blobs/backup.7z");
|
||||
|
||||
when(storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(31L, 7L)).thenReturn(Optional.of(archive));
|
||||
when(userRepository.findById(7L)).thenReturn(Optional.of(user));
|
||||
when(fileService.readZipCompatibleArchive(archive))
|
||||
.thenThrow(new BusinessException(ErrorCode.UNKNOWN, "压缩包读取失败"));
|
||||
|
||||
assertThatThrownBy(() -> handler.handle(createExtractTask(31L, 7L, "backup")))
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessage("extract task only supports zip-compatible archives");
|
||||
|
||||
verify(fileService, never()).importExternalFilesAtomically(any(), any(), any(), any());
|
||||
}
|
||||
|
||||
private BackgroundTask createExtractTask(Long fileId, Long userId, String outputDirectoryName) {
|
||||
BackgroundTask task = new BackgroundTask();
|
||||
task.setId(401L);
|
||||
task.setType(BackgroundTaskType.EXTRACT);
|
||||
task.setStatus(BackgroundTaskStatus.RUNNING);
|
||||
task.setUserId(userId);
|
||||
task.setPublicStateJson("""
|
||||
{"fileId":%d,"outputPath":"/docs","outputDirectoryName":"%s"}
|
||||
""".formatted(fileId, outputDirectoryName));
|
||||
task.setPrivateStateJson("""
|
||||
{"fileId":%d,"taskType":"EXTRACT","outputPath":"/docs","outputDirectoryName":"%s"}
|
||||
""".formatted(fileId, outputDirectoryName));
|
||||
return task;
|
||||
}
|
||||
|
||||
private User createUser(Long id) {
|
||||
User user = new User();
|
||||
user.setId(id);
|
||||
user.setUsername("alice");
|
||||
return user;
|
||||
}
|
||||
|
||||
private StoredFile createArchiveFile(Long id,
|
||||
User user,
|
||||
String path,
|
||||
String filename,
|
||||
String contentType,
|
||||
String objectKey) {
|
||||
StoredFile file = new StoredFile();
|
||||
file.setId(id);
|
||||
file.setUser(user);
|
||||
file.setPath(path);
|
||||
file.setFilename(filename);
|
||||
file.setDirectory(false);
|
||||
file.setContentType(contentType);
|
||||
file.setSize(12L);
|
||||
FileBlob blob = new FileBlob();
|
||||
blob.setId(id + 1000);
|
||||
blob.setObjectKey(objectKey);
|
||||
blob.setContentType(contentType);
|
||||
blob.setSize(12L);
|
||||
file.setBlob(blob);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.tasks;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.yoyuzh.files.core.FileBlob;
|
||||
import com.yoyuzh.files.core.StoredFile;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.search.FileMetadata;
|
||||
import com.yoyuzh.files.search.FileMetadataRepository;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -122,10 +127,10 @@ class MediaMetadataBackgroundTaskHandlerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldKeepNoopHandlerLimitedToArchiveAndExtract() {
|
||||
void shouldKeepNoopHandlerOutOfArchiveExtractAndMediaMetadata() {
|
||||
NoopBackgroundTaskHandler noop = new NoopBackgroundTaskHandler();
|
||||
assertThat(noop.supports(BackgroundTaskType.ARCHIVE)).isTrue();
|
||||
assertThat(noop.supports(BackgroundTaskType.EXTRACT)).isTrue();
|
||||
assertThat(noop.supports(BackgroundTaskType.ARCHIVE)).isFalse();
|
||||
assertThat(noop.supports(BackgroundTaskType.EXTRACT)).isFalse();
|
||||
assertThat(noop.supports(BackgroundTaskType.MEDIA_META)).isFalse();
|
||||
}
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
package com.yoyuzh.files;
|
||||
package com.yoyuzh.files.upload;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import com.yoyuzh.common.BusinessException;
|
||||
import com.yoyuzh.config.FileStorageProperties;
|
||||
import com.yoyuzh.files.core.FileService;
|
||||
import com.yoyuzh.files.core.StoredFileRepository;
|
||||
import com.yoyuzh.files.policy.StoragePolicy;
|
||||
import com.yoyuzh.files.policy.StoragePolicyCapabilities;
|
||||
import com.yoyuzh.files.policy.StoragePolicyService;
|
||||
import com.yoyuzh.files.policy.StoragePolicyType;
|
||||
import com.yoyuzh.files.storage.FileContentStorage;
|
||||
import com.yoyuzh.files.storage.PreparedUpload;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -15,6 +22,7 @@ import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.Map;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@@ -61,7 +69,20 @@ class UploadSessionServiceTest {
|
||||
void shouldCreateUploadSessionWithoutChangingLegacyUploadPath() {
|
||||
User user = createUser(7L);
|
||||
when(storedFileRepository.existsByUserIdAndPathAndFilename(7L, "/docs", "movie.mp4")).thenReturn(false);
|
||||
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy());
|
||||
StoragePolicy policy = createDefaultStoragePolicy();
|
||||
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(policy);
|
||||
when(storagePolicyService.readCapabilities(policy)).thenReturn(new StoragePolicyCapabilities(
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
500L * 1024 * 1024
|
||||
));
|
||||
when(fileContentStorage.createMultipartUpload(any(), eq("video/mp4"))).thenReturn("upload-123");
|
||||
when(uploadSessionRepository.save(any(UploadSession.class))).thenAnswer(invocation -> {
|
||||
UploadSession session = invocation.getArgument(0);
|
||||
session.setId(100L);
|
||||
@@ -75,6 +96,7 @@ class UploadSessionServiceTest {
|
||||
|
||||
assertThat(session.getSessionId()).isNotBlank();
|
||||
assertThat(session.getObjectKey()).startsWith("blobs/");
|
||||
assertThat(session.getMultipartUploadId()).isEqualTo("upload-123");
|
||||
assertThat(session.getStatus()).isEqualTo(UploadSessionStatus.CREATED);
|
||||
assertThat(session.getStoragePolicyId()).isEqualTo(42L);
|
||||
assertThat(session.getChunkSize()).isEqualTo(8L * 1024 * 1024);
|
||||
@@ -82,6 +104,36 @@ class UploadSessionServiceTest {
|
||||
assertThat(session.getExpiresAt()).isEqualTo(LocalDateTime.of(2026, 4, 9, 6, 0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPrepareMultipartPartUploadForOwnedSession() {
|
||||
User user = createUser(7L);
|
||||
UploadSession session = createSession(user);
|
||||
session.setMultipartUploadId("upload-123");
|
||||
session.setChunkCount(3);
|
||||
session.setChunkSize(8L * 1024 * 1024);
|
||||
session.setSize(20L * 1024 * 1024);
|
||||
when(uploadSessionRepository.findBySessionIdAndUserId("session-1", 7L))
|
||||
.thenReturn(Optional.of(session));
|
||||
when(fileContentStorage.prepareMultipartPartUpload(
|
||||
"blobs/session-1",
|
||||
"upload-123",
|
||||
3,
|
||||
"video/mp4",
|
||||
4L * 1024 * 1024
|
||||
)).thenReturn(new PreparedUpload(
|
||||
true,
|
||||
"https://upload.example.com/session-1/part-3",
|
||||
"PUT",
|
||||
Map.of("Content-Type", "video/mp4"),
|
||||
"blobs/session-1"
|
||||
));
|
||||
|
||||
PreparedUpload preparedUpload = uploadSessionService.prepareOwnedPartUpload(user, "session-1", 2);
|
||||
|
||||
assertThat(preparedUpload.uploadUrl()).isEqualTo("https://upload.example.com/session-1/part-3");
|
||||
assertThat(preparedUpload.method()).isEqualTo("PUT");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldOnlyReturnSessionOwnedByCurrentUser() {
|
||||
User user = createUser(7L);
|
||||
@@ -112,6 +164,15 @@ class UploadSessionServiceTest {
|
||||
void shouldCompleteOwnedSessionThroughLegacyFileCommitPath() {
|
||||
User user = createUser(7L);
|
||||
UploadSession session = createSession(user);
|
||||
session.setMultipartUploadId("upload-123");
|
||||
session.setChunkCount(2);
|
||||
session.setChunkSize(8L * 1024 * 1024);
|
||||
session.setUploadedPartsJson("""
|
||||
[
|
||||
{"partIndex":0,"etag":"etag-1","size":8388608,"uploadedAt":"2026-04-08T06:00:00"},
|
||||
{"partIndex":1,"etag":"etag-2","size":12,"uploadedAt":"2026-04-08T06:01:00"}
|
||||
]
|
||||
""");
|
||||
when(uploadSessionRepository.findBySessionIdAndUserId("session-1", 7L))
|
||||
.thenReturn(Optional.of(session));
|
||||
when(uploadSessionRepository.save(any(UploadSession.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
@@ -120,6 +181,7 @@ class UploadSessionServiceTest {
|
||||
|
||||
assertThat(result.getStatus()).isEqualTo(UploadSessionStatus.COMPLETED);
|
||||
assertThat(result.getUpdatedAt()).isEqualTo(LocalDateTime.of(2026, 4, 8, 6, 0));
|
||||
verify(fileContentStorage).completeMultipartUpload(eq("blobs/session-1"), eq("upload-123"), anyList());
|
||||
ArgumentCaptor<CompleteUploadRequest> requestCaptor = ArgumentCaptor.forClass(CompleteUploadRequest.class);
|
||||
verify(fileService).completeUpload(eq(user), requestCaptor.capture());
|
||||
assertThat(requestCaptor.getValue().path()).isEqualTo("/docs");
|
||||
@@ -196,6 +258,7 @@ class UploadSessionServiceTest {
|
||||
UploadSession session = createSession(user);
|
||||
session.setStatus(UploadSessionStatus.UPLOADING);
|
||||
session.setObjectKey("blobs/expired-session");
|
||||
session.setMultipartUploadId("upload-expired");
|
||||
session.setExpiresAt(LocalDateTime.of(2026, 4, 8, 5, 0));
|
||||
when(uploadSessionRepository.findByStatusInAndExpiresAtBefore(anyList(), eq(LocalDateTime.of(2026, 4, 8, 6, 0))))
|
||||
.thenReturn(List.of(session));
|
||||
@@ -205,7 +268,7 @@ class UploadSessionServiceTest {
|
||||
assertThat(expiredCount).isEqualTo(1);
|
||||
assertThat(session.getStatus()).isEqualTo(UploadSessionStatus.EXPIRED);
|
||||
assertThat(session.getUpdatedAt()).isEqualTo(LocalDateTime.of(2026, 4, 8, 6, 0));
|
||||
verify(fileContentStorage).deleteBlob("blobs/expired-session");
|
||||
verify(fileContentStorage).abortMultipartUpload("blobs/expired-session", "upload-expired");
|
||||
verify(uploadSessionRepository).saveAll(List.of(session));
|
||||
}
|
||||
|
||||
@@ -238,6 +301,7 @@ class UploadSessionServiceTest {
|
||||
session.setContentType("video/mp4");
|
||||
session.setSize(20L);
|
||||
session.setObjectKey("blobs/session-1");
|
||||
session.setMultipartUploadId(null);
|
||||
session.setChunkSize(8L * 1024 * 1024);
|
||||
session.setChunkCount(1);
|
||||
session.setUploadedPartsJson("[]");
|
||||
Reference in New Issue
Block a user