From 5802f396c527d8411de8d42b97136feee3970cfd Mon Sep 17 00:00:00 2001 From: yoyuzh Date: Wed, 8 Apr 2026 15:02:42 +0800 Subject: [PATCH] feat(files): add file entity migration --- .../java/com/yoyuzh/files/FileEntity.java | 163 ++++++++++++++++++ .../files/FileEntityBackfillService.java | 64 +++++++ .../yoyuzh/files/FileEntityRepository.java | 10 ++ .../java/com/yoyuzh/files/FileEntityType.java | 9 + .../java/com/yoyuzh/files/FileService.java | 91 +++++++++- .../java/com/yoyuzh/files/StoredFile.java | 26 +++ .../com/yoyuzh/files/StoredFileEntity.java | 92 ++++++++++ .../files/StoredFileEntityRepository.java | 6 + .../yoyuzh/files/StoredFileRepository.java | 3 + .../files/FileEntityBackfillServiceTest.java | 114 ++++++++++++ .../com/yoyuzh/files/FileServiceTest.java | 89 ++++++++++ docs/api-reference.md | 12 ++ docs/architecture.md | 13 ++ memory.md | 11 ++ 14 files changed, 699 insertions(+), 4 deletions(-) create mode 100644 backend/src/main/java/com/yoyuzh/files/FileEntity.java create mode 100644 backend/src/main/java/com/yoyuzh/files/FileEntityBackfillService.java create mode 100644 backend/src/main/java/com/yoyuzh/files/FileEntityRepository.java create mode 100644 backend/src/main/java/com/yoyuzh/files/FileEntityType.java create mode 100644 backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java create mode 100644 backend/src/main/java/com/yoyuzh/files/StoredFileEntityRepository.java create mode 100644 backend/src/test/java/com/yoyuzh/files/FileEntityBackfillServiceTest.java diff --git a/backend/src/main/java/com/yoyuzh/files/FileEntity.java b/backend/src/main/java/com/yoyuzh/files/FileEntity.java new file mode 100644 index 0000000..5f19643 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/FileEntity.java @@ -0,0 +1,163 @@ +package com.yoyuzh.files; + +import com.yoyuzh.auth.User; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.FetchType; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.PrePersist; +import jakarta.persistence.Table; +import org.hibernate.annotations.OnDelete; +import org.hibernate.annotations.OnDeleteAction; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "portal_file_entity", indexes = { + @Index(name = "uk_file_entity_key_type", columnList = "object_key,entity_type", unique = true), + @Index(name = "idx_file_entity_created_at", columnList = "created_at") +}) +public class FileEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "object_key", nullable = false, length = 512) + private String objectKey; + + @Column(nullable = false) + private Long size; + + @Column(name = "content_type", length = 255) + private String contentType; + + @Enumerated(EnumType.STRING) + @Column(name = "entity_type", nullable = false, length = 32) + private FileEntityType entityType; + + @Column(name = "reference_count", nullable = false) + private Integer referenceCount; + + @Column(name = "storage_policy_id") + private Long storagePolicyId; + + @Column(name = "upload_session_id", length = 64) + private String uploadSessionId; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "created_by") + @OnDelete(action = OnDeleteAction.SET_NULL) + private User createdBy; + + @Column(name = "props_json", columnDefinition = "TEXT") + private String propsJson; + + @Column(name = "created_at", nullable = false) + private LocalDateTime createdAt; + + @PrePersist + public void prePersist() { + if (createdAt == null) { + createdAt = LocalDateTime.now(); + } + if (referenceCount == null) { + referenceCount = 0; + } + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getObjectKey() { + return objectKey; + } + + public void setObjectKey(String objectKey) { + this.objectKey = objectKey; + } + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } + + public String getContentType() { + return contentType; + } + + public void setContentType(String contentType) { + this.contentType = contentType; + } + + public FileEntityType getEntityType() { + return entityType; + } + + public void setEntityType(FileEntityType entityType) { + this.entityType = entityType; + } + + public Integer getReferenceCount() { + return referenceCount; + } + + public void setReferenceCount(Integer referenceCount) { + this.referenceCount = referenceCount; + } + + public Long getStoragePolicyId() { + return storagePolicyId; + } + + public void setStoragePolicyId(Long storagePolicyId) { + this.storagePolicyId = storagePolicyId; + } + + public String getUploadSessionId() { + return uploadSessionId; + } + + public void setUploadSessionId(String uploadSessionId) { + this.uploadSessionId = uploadSessionId; + } + + public User getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(User createdBy) { + this.createdBy = createdBy; + } + + public String getPropsJson() { + return propsJson; + } + + public void setPropsJson(String propsJson) { + this.propsJson = propsJson; + } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(LocalDateTime createdAt) { + this.createdAt = createdAt; + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/FileEntityBackfillService.java b/backend/src/main/java/com/yoyuzh/files/FileEntityBackfillService.java new file mode 100644 index 0000000..1129e4f --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/FileEntityBackfillService.java @@ -0,0 +1,64 @@ +package com.yoyuzh.files; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.CommandLineRunner; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Optional; + +@Component +@Order(1) +@RequiredArgsConstructor +public class FileEntityBackfillService implements CommandLineRunner { + + static final String PRIMARY_ENTITY_ROLE = "PRIMARY"; + + private final StoredFileRepository storedFileRepository; + private final FileEntityRepository fileEntityRepository; + private final StoredFileEntityRepository storedFileEntityRepository; + + @Override + @Transactional + public void run(String... args) { + backfillPrimaryEntities(); + } + + @Transactional + public void backfillPrimaryEntities() { + for (StoredFile storedFile : storedFileRepository.findAllByDirectoryFalseAndBlobIsNotNullAndPrimaryEntityIsNull()) { + FileBlob blob = storedFile.getBlob(); + Optional existingEntity = fileEntityRepository + .findByObjectKeyAndEntityType(blob.getObjectKey(), FileEntityType.VERSION); + FileEntity fileEntity = existingEntity.orElseGet(() -> createEntity(storedFile, blob)); + + if (existingEntity.isPresent()) { + fileEntity.setReferenceCount(fileEntity.getReferenceCount() + 1); + fileEntityRepository.save(fileEntity); + } + storedFile.setPrimaryEntity(fileEntity); + storedFileRepository.save(storedFile); + storedFileEntityRepository.save(createRelation(storedFile, fileEntity)); + } + } + + private FileEntity createEntity(StoredFile storedFile, FileBlob blob) { + FileEntity fileEntity = new FileEntity(); + fileEntity.setObjectKey(blob.getObjectKey()); + fileEntity.setSize(blob.getSize()); + fileEntity.setContentType(blob.getContentType()); + fileEntity.setEntityType(FileEntityType.VERSION); + fileEntity.setReferenceCount(1); + fileEntity.setCreatedBy(storedFile.getUser()); + return fileEntityRepository.save(fileEntity); + } + + private StoredFileEntity createRelation(StoredFile storedFile, FileEntity fileEntity) { + StoredFileEntity relation = new StoredFileEntity(); + relation.setStoredFile(storedFile); + relation.setFileEntity(fileEntity); + relation.setEntityRole(PRIMARY_ENTITY_ROLE); + return relation; + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/FileEntityRepository.java b/backend/src/main/java/com/yoyuzh/files/FileEntityRepository.java new file mode 100644 index 0000000..ee073d3 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/FileEntityRepository.java @@ -0,0 +1,10 @@ +package com.yoyuzh.files; + +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; + +public interface FileEntityRepository extends JpaRepository { + + Optional findByObjectKeyAndEntityType(String objectKey, FileEntityType entityType); +} diff --git a/backend/src/main/java/com/yoyuzh/files/FileEntityType.java b/backend/src/main/java/com/yoyuzh/files/FileEntityType.java new file mode 100644 index 0000000..b961550 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/FileEntityType.java @@ -0,0 +1,9 @@ +package com.yoyuzh.files; + +public enum FileEntityType { + VERSION, + THUMBNAIL, + LIVE_PHOTO, + TRANSCODE, + AVATAR +} diff --git a/backend/src/main/java/com/yoyuzh/files/FileService.java b/backend/src/main/java/com/yoyuzh/files/FileService.java index 4835a82..79cad4f 100644 --- a/backend/src/main/java/com/yoyuzh/files/FileService.java +++ b/backend/src/main/java/com/yoyuzh/files/FileService.java @@ -51,6 +51,8 @@ public class FileService { private final StoredFileRepository storedFileRepository; private final FileBlobRepository fileBlobRepository; + private final FileEntityRepository fileEntityRepository; + private final StoredFileEntityRepository storedFileEntityRepository; private final FileContentStorage fileContentStorage; private final FileShareLinkRepository fileShareLinkRepository; private final AdminMetricsService adminMetricsService; @@ -63,15 +65,19 @@ public class FileService { @Autowired public FileService(StoredFileRepository storedFileRepository, FileBlobRepository fileBlobRepository, + FileEntityRepository fileEntityRepository, + StoredFileEntityRepository storedFileEntityRepository, FileContentStorage fileContentStorage, FileShareLinkRepository fileShareLinkRepository, AdminMetricsService adminMetricsService, FileStorageProperties properties) { - this(storedFileRepository, fileBlobRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC()); + this(storedFileRepository, fileBlobRepository, fileEntityRepository, storedFileEntityRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC()); } FileService(StoredFileRepository storedFileRepository, FileBlobRepository fileBlobRepository, + FileEntityRepository fileEntityRepository, + StoredFileEntityRepository storedFileEntityRepository, FileContentStorage fileContentStorage, FileShareLinkRepository fileShareLinkRepository, AdminMetricsService adminMetricsService, @@ -79,6 +85,8 @@ public class FileService { Clock clock) { this.storedFileRepository = storedFileRepository; this.fileBlobRepository = fileBlobRepository; + this.fileEntityRepository = fileEntityRepository; + this.storedFileEntityRepository = storedFileEntityRepository; this.fileContentStorage = fileContentStorage; this.fileShareLinkRepository = fileShareLinkRepository; this.adminMetricsService = adminMetricsService; @@ -93,6 +101,25 @@ public class FileService { this.clock = clock; } + FileService(StoredFileRepository storedFileRepository, + FileBlobRepository fileBlobRepository, + FileContentStorage fileContentStorage, + FileShareLinkRepository fileShareLinkRepository, + AdminMetricsService adminMetricsService, + FileStorageProperties properties) { + this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC()); + } + + FileService(StoredFileRepository storedFileRepository, + FileBlobRepository fileBlobRepository, + FileContentStorage fileContentStorage, + FileShareLinkRepository fileShareLinkRepository, + AdminMetricsService adminMetricsService, + FileStorageProperties properties, + Clock clock) { + this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, clock); + } + @Transactional public FileMetadataResponse upload(User user, String path, MultipartFile multipartFile) { String normalizedPath = normalizeDirectoryPath(path); @@ -349,7 +376,7 @@ public class FileService { if (!storedFile.isDirectory()) { ensureWithinStorageQuota(user, storedFile.getSize()); - return toResponse(storedFileRepository.save(copyStoredFile(storedFile, user, normalizedTargetPath))); + return toResponse(saveCopiedStoredFile(copyStoredFile(storedFile, user, normalizedTargetPath), user)); } String oldLogicalPath = buildLogicalPath(storedFile); @@ -385,7 +412,7 @@ public class FileService { StoredFile savedRoot = null; for (StoredFile copiedEntry : copiedEntries) { - StoredFile savedEntry = storedFileRepository.save(copiedEntry); + StoredFile savedEntry = saveCopiedStoredFile(copiedEntry, user); if (savedRoot == null) { savedRoot = savedEntry; } @@ -685,7 +712,52 @@ public class FileService { storedFile.setSize(size); storedFile.setDirectory(false); storedFile.setBlob(blob); - return toResponse(storedFileRepository.save(storedFile)); + FileEntity primaryEntity = createOrReferencePrimaryEntity(user, blob); + storedFile.setPrimaryEntity(primaryEntity); + StoredFile savedFile = storedFileRepository.save(storedFile); + savePrimaryEntityRelation(savedFile, primaryEntity); + return toResponse(savedFile); + } + + private FileEntity createOrReferencePrimaryEntity(User user, FileBlob blob) { + if (fileEntityRepository == null) { + return createTransientPrimaryEntity(user, blob); + } + + Optional existingEntity = fileEntityRepository.findByObjectKeyAndEntityType( + blob.getObjectKey(), + FileEntityType.VERSION + ); + if (existingEntity.isPresent()) { + FileEntity entity = existingEntity.get(); + entity.setReferenceCount(entity.getReferenceCount() + 1); + return fileEntityRepository.save(entity); + } + + return fileEntityRepository.save(createTransientPrimaryEntity(user, blob)); + } + + private FileEntity createTransientPrimaryEntity(User user, FileBlob blob) { + FileEntity entity = new FileEntity(); + entity.setObjectKey(blob.getObjectKey()); + entity.setContentType(blob.getContentType()); + entity.setSize(blob.getSize()); + entity.setEntityType(FileEntityType.VERSION); + entity.setReferenceCount(1); + entity.setCreatedBy(user); + return entity; + } + + private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) { + if (storedFileEntityRepository == null) { + return; + } + + StoredFileEntity relation = new StoredFileEntity(); + relation.setStoredFile(storedFile); + relation.setFileEntity(primaryEntity); + relation.setEntityRole("PRIMARY"); + storedFileEntityRepository.save(relation); } private FileShareLink getShareLink(String token) { @@ -961,6 +1033,17 @@ public class FileService { return copiedFile; } + private StoredFile saveCopiedStoredFile(StoredFile copiedFile, User owner) { + if (!copiedFile.isDirectory() && copiedFile.getBlob() != null && copiedFile.getPrimaryEntity() == null) { + copiedFile.setPrimaryEntity(createOrReferencePrimaryEntity(owner, copiedFile.getBlob())); + } + StoredFile savedFile = storedFileRepository.save(copiedFile); + if (!savedFile.isDirectory() && savedFile.getPrimaryEntity() != null) { + savePrimaryEntityRelation(savedFile, savedFile.getPrimaryEntity()); + } + return savedFile; + } + private String buildZipEntryName(String rootDirectoryName, String rootLogicalPath, StoredFile storedFile) { StringBuilder entryName = new StringBuilder(rootDirectoryName).append('/'); if (!storedFile.getPath().equals(rootLogicalPath)) { diff --git a/backend/src/main/java/com/yoyuzh/files/StoredFile.java b/backend/src/main/java/com/yoyuzh/files/StoredFile.java index 2a1bfc8..d2567bf 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoredFile.java +++ b/backend/src/main/java/com/yoyuzh/files/StoredFile.java @@ -42,6 +42,10 @@ public class StoredFile { @JoinColumn(name = "blob_id") private FileBlob blob; + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "primary_entity_id") + private FileEntity primaryEntity; + @Column(name = "storage_name", length = 255) private String legacyStorageName; @@ -57,6 +61,9 @@ public class StoredFile { @Column(name = "created_at", nullable = false) private LocalDateTime createdAt; + @Column(name = "updated_at") + private LocalDateTime updatedAt; + @Column(name = "deleted_at") private LocalDateTime deletedAt; @@ -74,6 +81,9 @@ public class StoredFile { if (createdAt == null) { createdAt = LocalDateTime.now(); } + if (updatedAt == null) { + updatedAt = createdAt; + } } public Long getId() { @@ -116,6 +126,14 @@ public class StoredFile { this.blob = blob; } + public FileEntity getPrimaryEntity() { + return primaryEntity; + } + + public void setPrimaryEntity(FileEntity primaryEntity) { + this.primaryEntity = primaryEntity; + } + public String getLegacyStorageName() { return legacyStorageName; } @@ -156,6 +174,14 @@ public class StoredFile { this.createdAt = createdAt; } + public LocalDateTime getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(LocalDateTime updatedAt) { + this.updatedAt = updatedAt; + } + public LocalDateTime getDeletedAt() { return deletedAt; } diff --git a/backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java b/backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java new file mode 100644 index 0000000..8c1ce28 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java @@ -0,0 +1,92 @@ +package com.yoyuzh.files; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.PrePersist; +import jakarta.persistence.Table; +import org.hibernate.annotations.OnDelete; +import org.hibernate.annotations.OnDeleteAction; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "portal_stored_file_entity", indexes = { + @Index(name = "uk_stored_file_entity_role", columnList = "stored_file_id,file_entity_id,entity_role", unique = true), + @Index(name = "idx_stored_file_entity_file", columnList = "stored_file_id"), + @Index(name = "idx_stored_file_entity_entity", columnList = "file_entity_id") +}) +public class StoredFileEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "stored_file_id", nullable = false) + @OnDelete(action = OnDeleteAction.CASCADE) + private StoredFile storedFile; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "file_entity_id", nullable = false) + private FileEntity fileEntity; + + @Column(name = "entity_role", nullable = false, length = 32) + private String entityRole; + + @Column(name = "created_at", nullable = false) + private LocalDateTime createdAt; + + @PrePersist + public void prePersist() { + if (createdAt == null) { + createdAt = LocalDateTime.now(); + } + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public StoredFile getStoredFile() { + return storedFile; + } + + public void setStoredFile(StoredFile storedFile) { + this.storedFile = storedFile; + } + + public FileEntity getFileEntity() { + return fileEntity; + } + + public void setFileEntity(FileEntity fileEntity) { + this.fileEntity = fileEntity; + } + + public String getEntityRole() { + return entityRole; + } + + public void setEntityRole(String entityRole) { + this.entityRole = entityRole; + } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(LocalDateTime createdAt) { + this.createdAt = createdAt; + } +} diff --git a/backend/src/main/java/com/yoyuzh/files/StoredFileEntityRepository.java b/backend/src/main/java/com/yoyuzh/files/StoredFileEntityRepository.java new file mode 100644 index 0000000..e63b4c9 --- /dev/null +++ b/backend/src/main/java/com/yoyuzh/files/StoredFileEntityRepository.java @@ -0,0 +1,6 @@ +package com.yoyuzh.files; + +import org.springframework.data.jpa.repository.JpaRepository; + +public interface StoredFileEntityRepository extends JpaRepository { +} diff --git a/backend/src/main/java/com/yoyuzh/files/StoredFileRepository.java b/backend/src/main/java/com/yoyuzh/files/StoredFileRepository.java index 35ef52f..9ef2947 100644 --- a/backend/src/main/java/com/yoyuzh/files/StoredFileRepository.java +++ b/backend/src/main/java/com/yoyuzh/files/StoredFileRepository.java @@ -123,4 +123,7 @@ public interface StoredFileRepository extends JpaRepository { Optional findDetailedById(@Param("id") Long id); List findAllByDirectoryFalseAndBlobIsNull(); + + @EntityGraph(attributePaths = {"user", "blob"}) + List findAllByDirectoryFalseAndBlobIsNotNullAndPrimaryEntityIsNull(); } diff --git a/backend/src/test/java/com/yoyuzh/files/FileEntityBackfillServiceTest.java b/backend/src/test/java/com/yoyuzh/files/FileEntityBackfillServiceTest.java new file mode 100644 index 0000000..dfffd54 --- /dev/null +++ b/backend/src/test/java/com/yoyuzh/files/FileEntityBackfillServiceTest.java @@ -0,0 +1,114 @@ +package com.yoyuzh.files; + +import com.yoyuzh.auth.User; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class FileEntityBackfillServiceTest { + + @Mock + private StoredFileRepository storedFileRepository; + @Mock + private FileEntityRepository fileEntityRepository; + @Mock + private StoredFileEntityRepository storedFileEntityRepository; + + private FileEntityBackfillService backfillService; + + @BeforeEach + void setUp() { + backfillService = new FileEntityBackfillService( + storedFileRepository, + fileEntityRepository, + storedFileEntityRepository + ); + } + + @Test + void shouldBackfillPrimaryEntityFromExistingBlob() { + StoredFile storedFile = createStoredFile(10L, 7L, "notes.txt", createBlob(20L, "blobs/blob-20")); + when(storedFileRepository.findAllByDirectoryFalseAndBlobIsNotNullAndPrimaryEntityIsNull()) + .thenReturn(List.of(storedFile)); + when(fileEntityRepository.findByObjectKeyAndEntityType("blobs/blob-20", FileEntityType.VERSION)) + .thenReturn(Optional.empty()); + when(fileEntityRepository.save(any(FileEntity.class))).thenAnswer(invocation -> { + FileEntity entity = invocation.getArgument(0); + entity.setId(100L); + return entity; + }); + + backfillService.backfillPrimaryEntities(); + + assertThat(storedFile.getPrimaryEntity()).isNotNull(); + assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo("blobs/blob-20"); + assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION); + assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1); + verify(fileEntityRepository).save(any(FileEntity.class)); + verify(storedFileRepository).save(storedFile); + verify(storedFileEntityRepository).save(any(StoredFileEntity.class)); + } + + @Test + void shouldReuseExistingFileEntityWhenBackfillRunsAgain() { + StoredFile storedFile = createStoredFile(11L, 8L, "report.pdf", createBlob(21L, "blobs/blob-21")); + FileEntity existingEntity = new FileEntity(); + existingEntity.setId(101L); + existingEntity.setObjectKey("blobs/blob-21"); + existingEntity.setEntityType(FileEntityType.VERSION); + existingEntity.setReferenceCount(3); + when(storedFileRepository.findAllByDirectoryFalseAndBlobIsNotNullAndPrimaryEntityIsNull()) + .thenReturn(List.of(storedFile)); + when(fileEntityRepository.findByObjectKeyAndEntityType("blobs/blob-21", FileEntityType.VERSION)) + .thenReturn(Optional.of(existingEntity)); + + backfillService.backfillPrimaryEntities(); + + assertThat(storedFile.getPrimaryEntity()).isSameAs(existingEntity); + assertThat(existingEntity.getReferenceCount()).isEqualTo(4); + verify(fileEntityRepository).save(existingEntity); + verify(storedFileRepository).save(storedFile); + verify(storedFileEntityRepository).save(any(StoredFileEntity.class)); + } + + private StoredFile createStoredFile(Long id, Long userId, String filename, FileBlob blob) { + User user = new User(); + user.setId(userId); + user.setUsername("user-" + userId); + + StoredFile file = new StoredFile(); + file.setId(id); + file.setUser(user); + file.setPath("/docs"); + file.setFilename(filename); + file.setBlob(blob); + file.setContentType(blob.getContentType()); + file.setSize(blob.getSize()); + file.setDirectory(false); + file.setCreatedAt(LocalDateTime.now()); + return file; + } + + private FileBlob createBlob(Long id, String objectKey) { + FileBlob blob = new FileBlob(); + blob.setId(id); + blob.setObjectKey(objectKey); + blob.setContentType("text/plain"); + blob.setSize(5L); + blob.setCreatedAt(LocalDateTime.now()); + return blob; + } +} diff --git a/backend/src/test/java/com/yoyuzh/files/FileServiceTest.java b/backend/src/test/java/com/yoyuzh/files/FileServiceTest.java index c825ee8..c367dc9 100644 --- a/backend/src/test/java/com/yoyuzh/files/FileServiceTest.java +++ b/backend/src/test/java/com/yoyuzh/files/FileServiceTest.java @@ -36,6 +36,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.AdditionalMatchers.aryEq; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentCaptor.forClass; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -50,6 +51,10 @@ class FileServiceTest { @Mock private FileBlobRepository fileBlobRepository; + @Mock + private FileEntityRepository fileEntityRepository; + @Mock + private StoredFileEntityRepository storedFileEntityRepository; @Mock private FileContentStorage fileContentStorage; @@ -104,6 +109,86 @@ class FileServiceTest { && "text/plain".equals(blob.getContentType()))); } + @Test + void shouldAttachPrimaryEntityWhenUploadingFile() { + User user = createUser(7L); + MockMultipartFile multipartFile = new MockMultipartFile( + "file", "notes.txt", "text/plain", "hello".getBytes()); + when(storedFileRepository.existsByUserIdAndPathAndFilename(7L, "/docs", "notes.txt")).thenReturn(false); + when(fileBlobRepository.save(any(FileBlob.class))).thenAnswer(invocation -> { + FileBlob blob = invocation.getArgument(0); + blob.setId(100L); + return blob; + }); + when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> { + StoredFile file = invocation.getArgument(0); + file.setId(10L); + return file; + }); + + fileService.upload(user, "/docs", multipartFile); + + var savedFileCaptor = forClass(StoredFile.class); + verify(storedFileRepository, times(2)).save(savedFileCaptor.capture()); + StoredFile storedFile = savedFileCaptor.getAllValues().stream() + .filter(file -> !file.isDirectory()) + .findFirst() + .orElseThrow(); + assertThat(storedFile.getPrimaryEntity()).isNotNull(); + assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo(storedFile.getBlob().getObjectKey()); + assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION); + assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1); + } + + @Test + void shouldPersistFileEntityAndRelationWhenUploadingFile() { + fileService = new FileService( + storedFileRepository, + fileBlobRepository, + fileEntityRepository, + storedFileEntityRepository, + fileContentStorage, + fileShareLinkRepository, + adminMetricsService, + new FileStorageProperties() + ); + User user = createUser(7L); + MockMultipartFile multipartFile = new MockMultipartFile( + "file", "notes.txt", "text/plain", "hello".getBytes()); + when(storedFileRepository.existsByUserIdAndPathAndFilename(7L, "/docs", "notes.txt")).thenReturn(false); + when(fileBlobRepository.save(any(FileBlob.class))).thenAnswer(invocation -> { + FileBlob blob = invocation.getArgument(0); + blob.setId(100L); + return blob; + }); + when(fileEntityRepository.findByObjectKeyAndEntityType(org.mockito.ArgumentMatchers.anyString(), eq(FileEntityType.VERSION))) + .thenReturn(Optional.empty()); + when(fileEntityRepository.save(any(FileEntity.class))).thenAnswer(invocation -> { + FileEntity entity = invocation.getArgument(0); + entity.setId(200L); + return entity; + }); + when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> { + StoredFile file = invocation.getArgument(0); + file.setId(10L); + return file; + }); + + fileService.upload(user, "/docs", multipartFile); + + var entityCaptor = forClass(FileEntity.class); + verify(fileEntityRepository).save(entityCaptor.capture()); + assertThat(entityCaptor.getValue().getObjectKey()).startsWith("blobs/"); + assertThat(entityCaptor.getValue().getEntityType()).isEqualTo(FileEntityType.VERSION); + assertThat(entityCaptor.getValue().getCreatedBy()).isSameAs(user); + + var relationCaptor = forClass(StoredFileEntity.class); + verify(storedFileEntityRepository).save(relationCaptor.capture()); + assertThat(relationCaptor.getValue().getStoredFile().getId()).isEqualTo(10L); + assertThat(relationCaptor.getValue().getFileEntity().getId()).isEqualTo(200L); + assertThat(relationCaptor.getValue().getEntityRole()).isEqualTo("PRIMARY"); + } + @Test void shouldInitiateDirectUploadThroughStorage() { User user = createUser(7L); @@ -323,6 +408,10 @@ class FileServiceTest { assertThat(response.id()).isEqualTo(20L); assertThat(response.path()).isEqualTo("/下载"); assertThat(file.getBlob()).isSameAs(blob); + var copiedFileCaptor = forClass(StoredFile.class); + verify(storedFileRepository).save(copiedFileCaptor.capture()); + assertThat(copiedFileCaptor.getValue().getPrimaryEntity()).isNotNull(); + assertThat(copiedFileCaptor.getValue().getPrimaryEntity().getObjectKey()).isEqualTo(blob.getObjectKey()); verify(fileContentStorage, never()).copyFile(any(), any(), any(), any()); } diff --git a/docs/api-reference.md b/docs/api-reference.md index 9d83852..badeec9 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -429,3 +429,15 @@ - 当前是 v2 API 的最小边界探针，返回结构为 `{ "code": 0, "msg": "success", "data": { "status": "ok", "apiVersion": "v2" } }`。 - v2 错误响应开始使用独立 `ApiV2ErrorCode` 范围；旧 `/api/**` 接口暂不迁移。 - 前端访问 v2 接口时可通过 `apiV2Request()` 自动拼接 `/api/v2/**`，内部请求会携带 `X-Yoyuzh-Client-Id`。 + +## 2026-04-08 文件实体模型二期第一小步 + +- 本阶段只新增后端实体和迁移映射，不新增对外 API。 +- 旧 `/api/files/**`、分享、回收站、快传接口继续使用现有 DTO 和响应结构。 +- `StoredFile.primaryEntity` 与 `portal_stored_file_entity` 目前只作为兼容迁移数据，后续阶段稳定后再切换新读写路径。 + +## 2026-04-08 文件实体模型二期第二小步 + +- 本阶段不新增对外 API，`/api/files/**`、分享、回收站、快传导入等响应结构保持不变。 +- 后端在旧接口内部开始双写实体模型：上传完成、外部导入、分享导入和网盘复制会继续写 `FileBlob`，同时创建或复用 `FileEntity.VERSION`，并写入 `StoredFile.primaryEntity` 与 `StoredFileEntity(PRIMARY)`。 +- 下载、分享详情、回收站、ZIP 下载仍读取 `StoredFile.blob`；后续阶段稳定后再切换到 `primaryEntity` 读取。 diff --git a/docs/architecture.md b/docs/architecture.md index 652d74d..c3a5722 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -432,3 +432,16 @@ Android 壳补充说明： - 后端新增 `com.yoyuzh.api.v2` 作为新版 API 的独立边界，当前只暴露公开健康检查 `GET /api/v2/site/ping`。 - v2 边界使用独立的 `ApiV2Response`、`ApiV2ErrorCode` 和 `ApiV2ExceptionHandler`，暂不替换旧 `com.yoyuzh.common.ApiResponse`。 - 前端 `front/src/lib/api.ts` 通过 `apiV2Request()` 访问 `/api/v2/**`，并为内部 API 请求附带稳定的 `X-Yoyuzh-Client-Id`，用于后续文件事件流和客户端事件去重。 + +## 2026-04-08 文件实体模型二期第一小步 + +- `StoredFile` 仍是用户可见文件/目录元数据的主模型，现阶段继续保留 `blob_id` 读取路径。 +- 新增 `FileEntity` 作为更通用的物理实体模型，当前先从 `FileBlob` 回填 `VERSION` 类型实体；后续版本、缩略图、转码、头像等派生对象会挂到同一实体体系。 +- 新增 `StoredFileEntity` 作为逻辑文件和物理/派生实体的关系表；当前只写入 `PRIMARY` 关系，不切换旧业务读写。 +- `FileEntityBackfillService` 在 `FileBlobBackfillService` 之后运行，只处理 `blob` 已存在但 `primaryEntity` 为空的普通文件，保证重复启动不会重复迁移已完成行。 + +## 2026-04-08 文件实体模型二期第二小步 + +- 文件写入路径已经从“只写 `FileBlob`”扩展为“继续写 `FileBlob`，同时写 `FileEntity.VERSION` 和 `StoredFileEntity(PRIMARY)`”。覆盖普通上传、直传完成、外部导入、分享导入和网盘复制。 +- `StoredFile.blob` 仍是当前生产读取路径；`StoredFile.primaryEntity` 与关系表暂时只作为兼容迁移数据，不影响旧 `/api/files/**` DTO 和前端调用。 +- `portal_stored_file_entity.stored_file_id` 随 `portal_file` 删除级联清理；`portal_file_entity.created_by` 在用户删除时置空，避免实体审计关系阻塞用户清理。 diff --git a/memory.md b/memory.md index a0081f8..bb40c7d 100644 --- a/memory.md +++ b/memory.md @@ -148,3 +148,14 @@ - 已按 Cloudreve 对照升级工程书落地第一阶段最小骨架：后端新增 `/api/v2/site/ping`、`ApiV2Response`、`ApiV2ErrorCode`、`ApiV2Exception` 与 v2 专用异常处理器，旧 `/api/**` 响应模型暂不替换。 - 前端 `front/src/lib/api.ts` 新增 `X-Yoyuzh-Client-Id` 约定和 `apiV2Request()`，内部 API 请求会携带稳定 client id；外部签名上传 URL 不携带该头。 - 修正 `.gitignore` 中 `storage/` 误忽略任意层级 `storage` 包的问题，改为只忽略仓库根 `/storage/` 和本地运行数据 `/backend/storage/`，否则 `backend/src/main/java/com/yoyuzh/files/storage/*` 会被误隐藏。 + +## 2026-04-08 阶段 2 第一小步记录 + +- 已新增文件实体模型二期的兼容表模型：`FileEntity`、`StoredFileEntity`、`FileEntityType`，并在 `StoredFile` 上新增 `primaryEntity` 与 `updatedAt`。 +- 已新增 `FileEntityBackfillService`，启动后在旧 `FileBlob` 仍保留的前提下，把已有 `StoredFile.blob` 只增量映射到 `FileEntity.VERSION` 与 `StoredFile.primaryEntity`；现有下载、复制、移动、分享、回收站读写路径暂不切换。 +- 当前阶段未删除 `FileBlob`，未切换前端，未引入上传会话二期。 +## 2026-04-08 阶段 2 第二小步记录 + +- 文件写入路径开始双写 `FileBlob + FileEntity.VERSION`：普通代理上传、直传完成、外部文件导入、分享导入，以及网盘复制复用 blob 时，都会给新 `StoredFile` 写入 `primaryEntity` 并创建 `StoredFileEntity(PRIMARY)` 关系。 +- 当前仍不切换读取路径：下载、ZIP、分享详情、回收站等旧业务继续依赖 `StoredFile.blob`，`primaryEntity` 只作为后续版本、缩略图、转码、存储策略迁移的兼容数据。 +- 为避免新关系表阻塞现有删除和测试清理，`StoredFileEntity -> StoredFile` 使用数据库级删除级联；`FileEntity.createdBy` 删除用户时置空，保留物理实体审计数据但不阻塞用户清理。