feat(files): add file entity migration
This commit is contained in:
163
backend/src/main/java/com/yoyuzh/files/FileEntity.java
Normal file
163
backend/src/main/java/com/yoyuzh/files/FileEntity.java
Normal file
@@ -0,0 +1,163 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EnumType;
|
||||
import jakarta.persistence.Enumerated;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Index;
|
||||
import jakarta.persistence.JoinColumn;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
import jakarta.persistence.PrePersist;
|
||||
import jakarta.persistence.Table;
|
||||
import org.hibernate.annotations.OnDelete;
|
||||
import org.hibernate.annotations.OnDeleteAction;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Entity
|
||||
@Table(name = "portal_file_entity", indexes = {
|
||||
@Index(name = "uk_file_entity_key_type", columnList = "object_key,entity_type", unique = true),
|
||||
@Index(name = "idx_file_entity_created_at", columnList = "created_at")
|
||||
})
|
||||
public class FileEntity {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column(name = "object_key", nullable = false, length = 512)
|
||||
private String objectKey;
|
||||
|
||||
@Column(nullable = false)
|
||||
private Long size;
|
||||
|
||||
@Column(name = "content_type", length = 255)
|
||||
private String contentType;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "entity_type", nullable = false, length = 32)
|
||||
private FileEntityType entityType;
|
||||
|
||||
@Column(name = "reference_count", nullable = false)
|
||||
private Integer referenceCount;
|
||||
|
||||
@Column(name = "storage_policy_id")
|
||||
private Long storagePolicyId;
|
||||
|
||||
@Column(name = "upload_session_id", length = 64)
|
||||
private String uploadSessionId;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "created_by")
|
||||
@OnDelete(action = OnDeleteAction.SET_NULL)
|
||||
private User createdBy;
|
||||
|
||||
@Column(name = "props_json", columnDefinition = "TEXT")
|
||||
private String propsJson;
|
||||
|
||||
@Column(name = "created_at", nullable = false)
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
@PrePersist
|
||||
public void prePersist() {
|
||||
if (createdAt == null) {
|
||||
createdAt = LocalDateTime.now();
|
||||
}
|
||||
if (referenceCount == null) {
|
||||
referenceCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(Long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getObjectKey() {
|
||||
return objectKey;
|
||||
}
|
||||
|
||||
public void setObjectKey(String objectKey) {
|
||||
this.objectKey = objectKey;
|
||||
}
|
||||
|
||||
public Long getSize() {
|
||||
return size;
|
||||
}
|
||||
|
||||
public void setSize(Long size) {
|
||||
this.size = size;
|
||||
}
|
||||
|
||||
public String getContentType() {
|
||||
return contentType;
|
||||
}
|
||||
|
||||
public void setContentType(String contentType) {
|
||||
this.contentType = contentType;
|
||||
}
|
||||
|
||||
public FileEntityType getEntityType() {
|
||||
return entityType;
|
||||
}
|
||||
|
||||
public void setEntityType(FileEntityType entityType) {
|
||||
this.entityType = entityType;
|
||||
}
|
||||
|
||||
public Integer getReferenceCount() {
|
||||
return referenceCount;
|
||||
}
|
||||
|
||||
public void setReferenceCount(Integer referenceCount) {
|
||||
this.referenceCount = referenceCount;
|
||||
}
|
||||
|
||||
public Long getStoragePolicyId() {
|
||||
return storagePolicyId;
|
||||
}
|
||||
|
||||
public void setStoragePolicyId(Long storagePolicyId) {
|
||||
this.storagePolicyId = storagePolicyId;
|
||||
}
|
||||
|
||||
public String getUploadSessionId() {
|
||||
return uploadSessionId;
|
||||
}
|
||||
|
||||
public void setUploadSessionId(String uploadSessionId) {
|
||||
this.uploadSessionId = uploadSessionId;
|
||||
}
|
||||
|
||||
public User getCreatedBy() {
|
||||
return createdBy;
|
||||
}
|
||||
|
||||
public void setCreatedBy(User createdBy) {
|
||||
this.createdBy = createdBy;
|
||||
}
|
||||
|
||||
public String getPropsJson() {
|
||||
return propsJson;
|
||||
}
|
||||
|
||||
public void setPropsJson(String propsJson) {
|
||||
this.propsJson = propsJson;
|
||||
}
|
||||
|
||||
public LocalDateTime getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
|
||||
public void setCreatedAt(LocalDateTime createdAt) {
|
||||
this.createdAt = createdAt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@Component
|
||||
@Order(1)
|
||||
@RequiredArgsConstructor
|
||||
public class FileEntityBackfillService implements CommandLineRunner {
|
||||
|
||||
static final String PRIMARY_ENTITY_ROLE = "PRIMARY";
|
||||
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final FileEntityRepository fileEntityRepository;
|
||||
private final StoredFileEntityRepository storedFileEntityRepository;
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public void run(String... args) {
|
||||
backfillPrimaryEntities();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void backfillPrimaryEntities() {
|
||||
for (StoredFile storedFile : storedFileRepository.findAllByDirectoryFalseAndBlobIsNotNullAndPrimaryEntityIsNull()) {
|
||||
FileBlob blob = storedFile.getBlob();
|
||||
Optional<FileEntity> existingEntity = fileEntityRepository
|
||||
.findByObjectKeyAndEntityType(blob.getObjectKey(), FileEntityType.VERSION);
|
||||
FileEntity fileEntity = existingEntity.orElseGet(() -> createEntity(storedFile, blob));
|
||||
|
||||
if (existingEntity.isPresent()) {
|
||||
fileEntity.setReferenceCount(fileEntity.getReferenceCount() + 1);
|
||||
fileEntityRepository.save(fileEntity);
|
||||
}
|
||||
storedFile.setPrimaryEntity(fileEntity);
|
||||
storedFileRepository.save(storedFile);
|
||||
storedFileEntityRepository.save(createRelation(storedFile, fileEntity));
|
||||
}
|
||||
}
|
||||
|
||||
private FileEntity createEntity(StoredFile storedFile, FileBlob blob) {
|
||||
FileEntity fileEntity = new FileEntity();
|
||||
fileEntity.setObjectKey(blob.getObjectKey());
|
||||
fileEntity.setSize(blob.getSize());
|
||||
fileEntity.setContentType(blob.getContentType());
|
||||
fileEntity.setEntityType(FileEntityType.VERSION);
|
||||
fileEntity.setReferenceCount(1);
|
||||
fileEntity.setCreatedBy(storedFile.getUser());
|
||||
return fileEntityRepository.save(fileEntity);
|
||||
}
|
||||
|
||||
private StoredFileEntity createRelation(StoredFile storedFile, FileEntity fileEntity) {
|
||||
StoredFileEntity relation = new StoredFileEntity();
|
||||
relation.setStoredFile(storedFile);
|
||||
relation.setFileEntity(fileEntity);
|
||||
relation.setEntityRole(PRIMARY_ENTITY_ROLE);
|
||||
return relation;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
public interface FileEntityRepository extends JpaRepository<FileEntity, Long> {
|
||||
|
||||
Optional<FileEntity> findByObjectKeyAndEntityType(String objectKey, FileEntityType entityType);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
public enum FileEntityType {
|
||||
VERSION,
|
||||
THUMBNAIL,
|
||||
LIVE_PHOTO,
|
||||
TRANSCODE,
|
||||
AVATAR
|
||||
}
|
||||
@@ -51,6 +51,8 @@ public class FileService {
|
||||
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final FileBlobRepository fileBlobRepository;
|
||||
private final FileEntityRepository fileEntityRepository;
|
||||
private final StoredFileEntityRepository storedFileEntityRepository;
|
||||
private final FileContentStorage fileContentStorage;
|
||||
private final FileShareLinkRepository fileShareLinkRepository;
|
||||
private final AdminMetricsService adminMetricsService;
|
||||
@@ -63,15 +65,19 @@ public class FileService {
|
||||
@Autowired
|
||||
public FileService(StoredFileRepository storedFileRepository,
|
||||
FileBlobRepository fileBlobRepository,
|
||||
FileEntityRepository fileEntityRepository,
|
||||
StoredFileEntityRepository storedFileEntityRepository,
|
||||
FileContentStorage fileContentStorage,
|
||||
FileShareLinkRepository fileShareLinkRepository,
|
||||
AdminMetricsService adminMetricsService,
|
||||
FileStorageProperties properties) {
|
||||
this(storedFileRepository, fileBlobRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC());
|
||||
this(storedFileRepository, fileBlobRepository, fileEntityRepository, storedFileEntityRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC());
|
||||
}
|
||||
|
||||
FileService(StoredFileRepository storedFileRepository,
|
||||
FileBlobRepository fileBlobRepository,
|
||||
FileEntityRepository fileEntityRepository,
|
||||
StoredFileEntityRepository storedFileEntityRepository,
|
||||
FileContentStorage fileContentStorage,
|
||||
FileShareLinkRepository fileShareLinkRepository,
|
||||
AdminMetricsService adminMetricsService,
|
||||
@@ -79,6 +85,8 @@ public class FileService {
|
||||
Clock clock) {
|
||||
this.storedFileRepository = storedFileRepository;
|
||||
this.fileBlobRepository = fileBlobRepository;
|
||||
this.fileEntityRepository = fileEntityRepository;
|
||||
this.storedFileEntityRepository = storedFileEntityRepository;
|
||||
this.fileContentStorage = fileContentStorage;
|
||||
this.fileShareLinkRepository = fileShareLinkRepository;
|
||||
this.adminMetricsService = adminMetricsService;
|
||||
@@ -93,6 +101,25 @@ public class FileService {
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
FileService(StoredFileRepository storedFileRepository,
|
||||
FileBlobRepository fileBlobRepository,
|
||||
FileContentStorage fileContentStorage,
|
||||
FileShareLinkRepository fileShareLinkRepository,
|
||||
AdminMetricsService adminMetricsService,
|
||||
FileStorageProperties properties) {
|
||||
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC());
|
||||
}
|
||||
|
||||
FileService(StoredFileRepository storedFileRepository,
|
||||
FileBlobRepository fileBlobRepository,
|
||||
FileContentStorage fileContentStorage,
|
||||
FileShareLinkRepository fileShareLinkRepository,
|
||||
AdminMetricsService adminMetricsService,
|
||||
FileStorageProperties properties,
|
||||
Clock clock) {
|
||||
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, clock);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public FileMetadataResponse upload(User user, String path, MultipartFile multipartFile) {
|
||||
String normalizedPath = normalizeDirectoryPath(path);
|
||||
@@ -349,7 +376,7 @@ public class FileService {
|
||||
|
||||
if (!storedFile.isDirectory()) {
|
||||
ensureWithinStorageQuota(user, storedFile.getSize());
|
||||
return toResponse(storedFileRepository.save(copyStoredFile(storedFile, user, normalizedTargetPath)));
|
||||
return toResponse(saveCopiedStoredFile(copyStoredFile(storedFile, user, normalizedTargetPath), user));
|
||||
}
|
||||
|
||||
String oldLogicalPath = buildLogicalPath(storedFile);
|
||||
@@ -385,7 +412,7 @@ public class FileService {
|
||||
|
||||
StoredFile savedRoot = null;
|
||||
for (StoredFile copiedEntry : copiedEntries) {
|
||||
StoredFile savedEntry = storedFileRepository.save(copiedEntry);
|
||||
StoredFile savedEntry = saveCopiedStoredFile(copiedEntry, user);
|
||||
if (savedRoot == null) {
|
||||
savedRoot = savedEntry;
|
||||
}
|
||||
@@ -685,7 +712,52 @@ public class FileService {
|
||||
storedFile.setSize(size);
|
||||
storedFile.setDirectory(false);
|
||||
storedFile.setBlob(blob);
|
||||
return toResponse(storedFileRepository.save(storedFile));
|
||||
FileEntity primaryEntity = createOrReferencePrimaryEntity(user, blob);
|
||||
storedFile.setPrimaryEntity(primaryEntity);
|
||||
StoredFile savedFile = storedFileRepository.save(storedFile);
|
||||
savePrimaryEntityRelation(savedFile, primaryEntity);
|
||||
return toResponse(savedFile);
|
||||
}
|
||||
|
||||
private FileEntity createOrReferencePrimaryEntity(User user, FileBlob blob) {
|
||||
if (fileEntityRepository == null) {
|
||||
return createTransientPrimaryEntity(user, blob);
|
||||
}
|
||||
|
||||
Optional<FileEntity> existingEntity = fileEntityRepository.findByObjectKeyAndEntityType(
|
||||
blob.getObjectKey(),
|
||||
FileEntityType.VERSION
|
||||
);
|
||||
if (existingEntity.isPresent()) {
|
||||
FileEntity entity = existingEntity.get();
|
||||
entity.setReferenceCount(entity.getReferenceCount() + 1);
|
||||
return fileEntityRepository.save(entity);
|
||||
}
|
||||
|
||||
return fileEntityRepository.save(createTransientPrimaryEntity(user, blob));
|
||||
}
|
||||
|
||||
private FileEntity createTransientPrimaryEntity(User user, FileBlob blob) {
|
||||
FileEntity entity = new FileEntity();
|
||||
entity.setObjectKey(blob.getObjectKey());
|
||||
entity.setContentType(blob.getContentType());
|
||||
entity.setSize(blob.getSize());
|
||||
entity.setEntityType(FileEntityType.VERSION);
|
||||
entity.setReferenceCount(1);
|
||||
entity.setCreatedBy(user);
|
||||
return entity;
|
||||
}
|
||||
|
||||
private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) {
|
||||
if (storedFileEntityRepository == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
StoredFileEntity relation = new StoredFileEntity();
|
||||
relation.setStoredFile(storedFile);
|
||||
relation.setFileEntity(primaryEntity);
|
||||
relation.setEntityRole("PRIMARY");
|
||||
storedFileEntityRepository.save(relation);
|
||||
}
|
||||
|
||||
private FileShareLink getShareLink(String token) {
|
||||
@@ -961,6 +1033,17 @@ public class FileService {
|
||||
return copiedFile;
|
||||
}
|
||||
|
||||
private StoredFile saveCopiedStoredFile(StoredFile copiedFile, User owner) {
|
||||
if (!copiedFile.isDirectory() && copiedFile.getBlob() != null && copiedFile.getPrimaryEntity() == null) {
|
||||
copiedFile.setPrimaryEntity(createOrReferencePrimaryEntity(owner, copiedFile.getBlob()));
|
||||
}
|
||||
StoredFile savedFile = storedFileRepository.save(copiedFile);
|
||||
if (!savedFile.isDirectory() && savedFile.getPrimaryEntity() != null) {
|
||||
savePrimaryEntityRelation(savedFile, savedFile.getPrimaryEntity());
|
||||
}
|
||||
return savedFile;
|
||||
}
|
||||
|
||||
private String buildZipEntryName(String rootDirectoryName, String rootLogicalPath, StoredFile storedFile) {
|
||||
StringBuilder entryName = new StringBuilder(rootDirectoryName).append('/');
|
||||
if (!storedFile.getPath().equals(rootLogicalPath)) {
|
||||
|
||||
@@ -42,6 +42,10 @@ public class StoredFile {
|
||||
@JoinColumn(name = "blob_id")
|
||||
private FileBlob blob;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "primary_entity_id")
|
||||
private FileEntity primaryEntity;
|
||||
|
||||
@Column(name = "storage_name", length = 255)
|
||||
private String legacyStorageName;
|
||||
|
||||
@@ -57,6 +61,9 @@ public class StoredFile {
|
||||
@Column(name = "created_at", nullable = false)
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
@Column(name = "updated_at")
|
||||
private LocalDateTime updatedAt;
|
||||
|
||||
@Column(name = "deleted_at")
|
||||
private LocalDateTime deletedAt;
|
||||
|
||||
@@ -74,6 +81,9 @@ public class StoredFile {
|
||||
if (createdAt == null) {
|
||||
createdAt = LocalDateTime.now();
|
||||
}
|
||||
if (updatedAt == null) {
|
||||
updatedAt = createdAt;
|
||||
}
|
||||
}
|
||||
|
||||
public Long getId() {
|
||||
@@ -116,6 +126,14 @@ public class StoredFile {
|
||||
this.blob = blob;
|
||||
}
|
||||
|
||||
public FileEntity getPrimaryEntity() {
|
||||
return primaryEntity;
|
||||
}
|
||||
|
||||
public void setPrimaryEntity(FileEntity primaryEntity) {
|
||||
this.primaryEntity = primaryEntity;
|
||||
}
|
||||
|
||||
public String getLegacyStorageName() {
|
||||
return legacyStorageName;
|
||||
}
|
||||
@@ -156,6 +174,14 @@ public class StoredFile {
|
||||
this.createdAt = createdAt;
|
||||
}
|
||||
|
||||
public LocalDateTime getUpdatedAt() {
|
||||
return updatedAt;
|
||||
}
|
||||
|
||||
public void setUpdatedAt(LocalDateTime updatedAt) {
|
||||
this.updatedAt = updatedAt;
|
||||
}
|
||||
|
||||
public LocalDateTime getDeletedAt() {
|
||||
return deletedAt;
|
||||
}
|
||||
|
||||
92
backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java
Normal file
92
backend/src/main/java/com/yoyuzh/files/StoredFileEntity.java
Normal file
@@ -0,0 +1,92 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Index;
|
||||
import jakarta.persistence.JoinColumn;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
import jakarta.persistence.PrePersist;
|
||||
import jakarta.persistence.Table;
|
||||
import org.hibernate.annotations.OnDelete;
|
||||
import org.hibernate.annotations.OnDeleteAction;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Entity
|
||||
@Table(name = "portal_stored_file_entity", indexes = {
|
||||
@Index(name = "uk_stored_file_entity_role", columnList = "stored_file_id,file_entity_id,entity_role", unique = true),
|
||||
@Index(name = "idx_stored_file_entity_file", columnList = "stored_file_id"),
|
||||
@Index(name = "idx_stored_file_entity_entity", columnList = "file_entity_id")
|
||||
})
|
||||
public class StoredFileEntity {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY, optional = false)
|
||||
@JoinColumn(name = "stored_file_id", nullable = false)
|
||||
@OnDelete(action = OnDeleteAction.CASCADE)
|
||||
private StoredFile storedFile;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY, optional = false)
|
||||
@JoinColumn(name = "file_entity_id", nullable = false)
|
||||
private FileEntity fileEntity;
|
||||
|
||||
@Column(name = "entity_role", nullable = false, length = 32)
|
||||
private String entityRole;
|
||||
|
||||
@Column(name = "created_at", nullable = false)
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
@PrePersist
|
||||
public void prePersist() {
|
||||
if (createdAt == null) {
|
||||
createdAt = LocalDateTime.now();
|
||||
}
|
||||
}
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(Long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public StoredFile getStoredFile() {
|
||||
return storedFile;
|
||||
}
|
||||
|
||||
public void setStoredFile(StoredFile storedFile) {
|
||||
this.storedFile = storedFile;
|
||||
}
|
||||
|
||||
public FileEntity getFileEntity() {
|
||||
return fileEntity;
|
||||
}
|
||||
|
||||
public void setFileEntity(FileEntity fileEntity) {
|
||||
this.fileEntity = fileEntity;
|
||||
}
|
||||
|
||||
public String getEntityRole() {
|
||||
return entityRole;
|
||||
}
|
||||
|
||||
public void setEntityRole(String entityRole) {
|
||||
this.entityRole = entityRole;
|
||||
}
|
||||
|
||||
public LocalDateTime getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
|
||||
public void setCreatedAt(LocalDateTime createdAt) {
|
||||
this.createdAt = createdAt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
public interface StoredFileEntityRepository extends JpaRepository<StoredFileEntity, Long> {
|
||||
}
|
||||
@@ -123,4 +123,7 @@ public interface StoredFileRepository extends JpaRepository<StoredFile, Long> {
|
||||
Optional<StoredFile> findDetailedById(@Param("id") Long id);
|
||||
|
||||
List<StoredFile> findAllByDirectoryFalseAndBlobIsNull();
|
||||
|
||||
@EntityGraph(attributePaths = {"user", "blob"})
|
||||
List<StoredFile> findAllByDirectoryFalseAndBlobIsNotNullAndPrimaryEntityIsNull();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package com.yoyuzh.files;
|
||||
|
||||
import com.yoyuzh.auth.User;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class FileEntityBackfillServiceTest {
|
||||
|
||||
@Mock
|
||||
private StoredFileRepository storedFileRepository;
|
||||
@Mock
|
||||
private FileEntityRepository fileEntityRepository;
|
||||
@Mock
|
||||
private StoredFileEntityRepository storedFileEntityRepository;
|
||||
|
||||
private FileEntityBackfillService backfillService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
backfillService = new FileEntityBackfillService(
|
||||
storedFileRepository,
|
||||
fileEntityRepository,
|
||||
storedFileEntityRepository
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldBackfillPrimaryEntityFromExistingBlob() {
|
||||
StoredFile storedFile = createStoredFile(10L, 7L, "notes.txt", createBlob(20L, "blobs/blob-20"));
|
||||
when(storedFileRepository.findAllByDirectoryFalseAndBlobIsNotNullAndPrimaryEntityIsNull())
|
||||
.thenReturn(List.of(storedFile));
|
||||
when(fileEntityRepository.findByObjectKeyAndEntityType("blobs/blob-20", FileEntityType.VERSION))
|
||||
.thenReturn(Optional.empty());
|
||||
when(fileEntityRepository.save(any(FileEntity.class))).thenAnswer(invocation -> {
|
||||
FileEntity entity = invocation.getArgument(0);
|
||||
entity.setId(100L);
|
||||
return entity;
|
||||
});
|
||||
|
||||
backfillService.backfillPrimaryEntities();
|
||||
|
||||
assertThat(storedFile.getPrimaryEntity()).isNotNull();
|
||||
assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo("blobs/blob-20");
|
||||
assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION);
|
||||
assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1);
|
||||
verify(fileEntityRepository).save(any(FileEntity.class));
|
||||
verify(storedFileRepository).save(storedFile);
|
||||
verify(storedFileEntityRepository).save(any(StoredFileEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReuseExistingFileEntityWhenBackfillRunsAgain() {
|
||||
StoredFile storedFile = createStoredFile(11L, 8L, "report.pdf", createBlob(21L, "blobs/blob-21"));
|
||||
FileEntity existingEntity = new FileEntity();
|
||||
existingEntity.setId(101L);
|
||||
existingEntity.setObjectKey("blobs/blob-21");
|
||||
existingEntity.setEntityType(FileEntityType.VERSION);
|
||||
existingEntity.setReferenceCount(3);
|
||||
when(storedFileRepository.findAllByDirectoryFalseAndBlobIsNotNullAndPrimaryEntityIsNull())
|
||||
.thenReturn(List.of(storedFile));
|
||||
when(fileEntityRepository.findByObjectKeyAndEntityType("blobs/blob-21", FileEntityType.VERSION))
|
||||
.thenReturn(Optional.of(existingEntity));
|
||||
|
||||
backfillService.backfillPrimaryEntities();
|
||||
|
||||
assertThat(storedFile.getPrimaryEntity()).isSameAs(existingEntity);
|
||||
assertThat(existingEntity.getReferenceCount()).isEqualTo(4);
|
||||
verify(fileEntityRepository).save(existingEntity);
|
||||
verify(storedFileRepository).save(storedFile);
|
||||
verify(storedFileEntityRepository).save(any(StoredFileEntity.class));
|
||||
}
|
||||
|
||||
private StoredFile createStoredFile(Long id, Long userId, String filename, FileBlob blob) {
|
||||
User user = new User();
|
||||
user.setId(userId);
|
||||
user.setUsername("user-" + userId);
|
||||
|
||||
StoredFile file = new StoredFile();
|
||||
file.setId(id);
|
||||
file.setUser(user);
|
||||
file.setPath("/docs");
|
||||
file.setFilename(filename);
|
||||
file.setBlob(blob);
|
||||
file.setContentType(blob.getContentType());
|
||||
file.setSize(blob.getSize());
|
||||
file.setDirectory(false);
|
||||
file.setCreatedAt(LocalDateTime.now());
|
||||
return file;
|
||||
}
|
||||
|
||||
private FileBlob createBlob(Long id, String objectKey) {
|
||||
FileBlob blob = new FileBlob();
|
||||
blob.setId(id);
|
||||
blob.setObjectKey(objectKey);
|
||||
blob.setContentType("text/plain");
|
||||
blob.setSize(5L);
|
||||
blob.setCreatedAt(LocalDateTime.now());
|
||||
return blob;
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.AdditionalMatchers.aryEq;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentCaptor.forClass;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
@@ -50,6 +51,10 @@ class FileServiceTest {
|
||||
|
||||
@Mock
|
||||
private FileBlobRepository fileBlobRepository;
|
||||
@Mock
|
||||
private FileEntityRepository fileEntityRepository;
|
||||
@Mock
|
||||
private StoredFileEntityRepository storedFileEntityRepository;
|
||||
|
||||
@Mock
|
||||
private FileContentStorage fileContentStorage;
|
||||
@@ -104,6 +109,86 @@ class FileServiceTest {
|
||||
&& "text/plain".equals(blob.getContentType())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldAttachPrimaryEntityWhenUploadingFile() {
|
||||
User user = createUser(7L);
|
||||
MockMultipartFile multipartFile = new MockMultipartFile(
|
||||
"file", "notes.txt", "text/plain", "hello".getBytes());
|
||||
when(storedFileRepository.existsByUserIdAndPathAndFilename(7L, "/docs", "notes.txt")).thenReturn(false);
|
||||
when(fileBlobRepository.save(any(FileBlob.class))).thenAnswer(invocation -> {
|
||||
FileBlob blob = invocation.getArgument(0);
|
||||
blob.setId(100L);
|
||||
return blob;
|
||||
});
|
||||
when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> {
|
||||
StoredFile file = invocation.getArgument(0);
|
||||
file.setId(10L);
|
||||
return file;
|
||||
});
|
||||
|
||||
fileService.upload(user, "/docs", multipartFile);
|
||||
|
||||
var savedFileCaptor = forClass(StoredFile.class);
|
||||
verify(storedFileRepository, times(2)).save(savedFileCaptor.capture());
|
||||
StoredFile storedFile = savedFileCaptor.getAllValues().stream()
|
||||
.filter(file -> !file.isDirectory())
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
assertThat(storedFile.getPrimaryEntity()).isNotNull();
|
||||
assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo(storedFile.getBlob().getObjectKey());
|
||||
assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION);
|
||||
assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPersistFileEntityAndRelationWhenUploadingFile() {
|
||||
fileService = new FileService(
|
||||
storedFileRepository,
|
||||
fileBlobRepository,
|
||||
fileEntityRepository,
|
||||
storedFileEntityRepository,
|
||||
fileContentStorage,
|
||||
fileShareLinkRepository,
|
||||
adminMetricsService,
|
||||
new FileStorageProperties()
|
||||
);
|
||||
User user = createUser(7L);
|
||||
MockMultipartFile multipartFile = new MockMultipartFile(
|
||||
"file", "notes.txt", "text/plain", "hello".getBytes());
|
||||
when(storedFileRepository.existsByUserIdAndPathAndFilename(7L, "/docs", "notes.txt")).thenReturn(false);
|
||||
when(fileBlobRepository.save(any(FileBlob.class))).thenAnswer(invocation -> {
|
||||
FileBlob blob = invocation.getArgument(0);
|
||||
blob.setId(100L);
|
||||
return blob;
|
||||
});
|
||||
when(fileEntityRepository.findByObjectKeyAndEntityType(org.mockito.ArgumentMatchers.anyString(), eq(FileEntityType.VERSION)))
|
||||
.thenReturn(Optional.empty());
|
||||
when(fileEntityRepository.save(any(FileEntity.class))).thenAnswer(invocation -> {
|
||||
FileEntity entity = invocation.getArgument(0);
|
||||
entity.setId(200L);
|
||||
return entity;
|
||||
});
|
||||
when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> {
|
||||
StoredFile file = invocation.getArgument(0);
|
||||
file.setId(10L);
|
||||
return file;
|
||||
});
|
||||
|
||||
fileService.upload(user, "/docs", multipartFile);
|
||||
|
||||
var entityCaptor = forClass(FileEntity.class);
|
||||
verify(fileEntityRepository).save(entityCaptor.capture());
|
||||
assertThat(entityCaptor.getValue().getObjectKey()).startsWith("blobs/");
|
||||
assertThat(entityCaptor.getValue().getEntityType()).isEqualTo(FileEntityType.VERSION);
|
||||
assertThat(entityCaptor.getValue().getCreatedBy()).isSameAs(user);
|
||||
|
||||
var relationCaptor = forClass(StoredFileEntity.class);
|
||||
verify(storedFileEntityRepository).save(relationCaptor.capture());
|
||||
assertThat(relationCaptor.getValue().getStoredFile().getId()).isEqualTo(10L);
|
||||
assertThat(relationCaptor.getValue().getFileEntity().getId()).isEqualTo(200L);
|
||||
assertThat(relationCaptor.getValue().getEntityRole()).isEqualTo("PRIMARY");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldInitiateDirectUploadThroughStorage() {
|
||||
User user = createUser(7L);
|
||||
@@ -323,6 +408,10 @@ class FileServiceTest {
|
||||
assertThat(response.id()).isEqualTo(20L);
|
||||
assertThat(response.path()).isEqualTo("/下载");
|
||||
assertThat(file.getBlob()).isSameAs(blob);
|
||||
var copiedFileCaptor = forClass(StoredFile.class);
|
||||
verify(storedFileRepository).save(copiedFileCaptor.capture());
|
||||
assertThat(copiedFileCaptor.getValue().getPrimaryEntity()).isNotNull();
|
||||
assertThat(copiedFileCaptor.getValue().getPrimaryEntity().getObjectKey()).isEqualTo(blob.getObjectKey());
|
||||
verify(fileContentStorage, never()).copyFile(any(), any(), any(), any());
|
||||
}
|
||||
|
||||
|
||||
@@ -429,3 +429,15 @@
|
||||
- 当前是 v2 API 的最小边界探针,返回结构为 `{ "code": 0, "msg": "success", "data": { "status": "ok", "apiVersion": "v2" } }`。
|
||||
- v2 错误响应开始使用独立 `ApiV2ErrorCode` 范围;旧 `/api/**` 接口暂不迁移。
|
||||
- 前端访问 v2 接口时可通过 `apiV2Request()` 自动拼接 `/api/v2/**`,内部请求会携带 `X-Yoyuzh-Client-Id`。
|
||||
|
||||
## 2026-04-08 文件实体模型二期第一小步
|
||||
|
||||
- 本阶段只新增后端实体和迁移映射,不新增对外 API。
|
||||
- 旧 `/api/files/**`、分享、回收站、快传接口继续使用现有 DTO 和响应结构。
|
||||
- `StoredFile.primaryEntity` 与 `portal_stored_file_entity` 目前只作为兼容迁移数据,后续阶段稳定后再切换新读写路径。
|
||||
|
||||
## 2026-04-08 文件实体模型二期第二小步
|
||||
|
||||
- 本阶段不新增对外 API,`/api/files/**`、分享、回收站、快传导入等响应结构保持不变。
|
||||
- 后端在旧接口内部开始双写实体模型:上传完成、外部导入、分享导入和网盘复制会继续写 `FileBlob`,同时创建或复用 `FileEntity.VERSION`,并写入 `StoredFile.primaryEntity` 与 `StoredFileEntity(PRIMARY)`。
|
||||
- 下载、分享详情、回收站、ZIP 下载仍读取 `StoredFile.blob`;后续阶段稳定后再切换到 `primaryEntity` 读取。
|
||||
|
||||
@@ -432,3 +432,16 @@ Android 壳补充说明:
|
||||
- 后端新增 `com.yoyuzh.api.v2` 作为新版 API 的独立边界,当前只暴露公开健康检查 `GET /api/v2/site/ping`。
|
||||
- v2 边界使用独立的 `ApiV2Response`、`ApiV2ErrorCode` 和 `ApiV2ExceptionHandler`,暂不替换旧 `com.yoyuzh.common.ApiResponse`。
|
||||
- 前端 `front/src/lib/api.ts` 通过 `apiV2Request()` 访问 `/api/v2/**`,并为内部 API 请求附带稳定的 `X-Yoyuzh-Client-Id`,用于后续文件事件流和客户端事件去重。
|
||||
|
||||
## 2026-04-08 文件实体模型二期第一小步
|
||||
|
||||
- `StoredFile` 仍是用户可见文件/目录元数据的主模型,现阶段继续保留 `blob_id` 读取路径。
|
||||
- 新增 `FileEntity` 作为更通用的物理实体模型,当前先从 `FileBlob` 回填 `VERSION` 类型实体;后续版本、缩略图、转码、头像等派生对象会挂到同一实体体系。
|
||||
- 新增 `StoredFileEntity` 作为逻辑文件和物理/派生实体的关系表;当前只写入 `PRIMARY` 关系,不切换旧业务读写。
|
||||
- `FileEntityBackfillService` 在 `FileBlobBackfillService` 之后运行,只处理 `blob` 已存在但 `primaryEntity` 为空的普通文件,保证重复启动不会重复迁移已完成行。
|
||||
|
||||
## 2026-04-08 文件实体模型二期第二小步
|
||||
|
||||
- 文件写入路径已经从“只写 `FileBlob`”扩展为“继续写 `FileBlob`,同时写 `FileEntity.VERSION` 和 `StoredFileEntity(PRIMARY)`”。覆盖普通上传、直传完成、外部导入、分享导入和网盘复制。
|
||||
- `StoredFile.blob` 仍是当前生产读取路径;`StoredFile.primaryEntity` 与关系表暂时只作为兼容迁移数据,不影响旧 `/api/files/**` DTO 和前端调用。
|
||||
- `portal_stored_file_entity.stored_file_id` 随 `portal_file` 删除级联清理;`portal_file_entity.created_by` 在用户删除时置空,避免实体审计关系阻塞用户清理。
|
||||
|
||||
11
memory.md
11
memory.md
@@ -148,3 +148,14 @@
|
||||
- 已按 Cloudreve 对照升级工程书落地第一阶段最小骨架:后端新增 `/api/v2/site/ping`、`ApiV2Response`、`ApiV2ErrorCode`、`ApiV2Exception` 与 v2 专用异常处理器,旧 `/api/**` 响应模型暂不替换。
|
||||
- 前端 `front/src/lib/api.ts` 新增 `X-Yoyuzh-Client-Id` 约定和 `apiV2Request()`,内部 API 请求会携带稳定 client id;外部签名上传 URL 不携带该头。
|
||||
- 修正 `.gitignore` 中 `storage/` 误忽略任意层级 `storage` 包的问题,改为只忽略仓库根 `/storage/` 和本地运行数据 `/backend/storage/`,否则 `backend/src/main/java/com/yoyuzh/files/storage/*` 会被误隐藏。
|
||||
|
||||
## 2026-04-08 阶段 2 第一小步记录
|
||||
|
||||
- 已新增文件实体模型二期的兼容表模型:`FileEntity`、`StoredFileEntity`、`FileEntityType`,并在 `StoredFile` 上新增 `primaryEntity` 与 `updatedAt`。
|
||||
- 已新增 `FileEntityBackfillService`,启动后在旧 `FileBlob` 仍保留的前提下,把已有 `StoredFile.blob` 只增量映射到 `FileEntity.VERSION` 与 `StoredFile.primaryEntity`;现有下载、复制、移动、分享、回收站读写路径暂不切换。
|
||||
- 当前阶段未删除 `FileBlob`,未切换前端,未引入上传会话二期。
|
||||
## 2026-04-08 阶段 2 第二小步记录
|
||||
|
||||
- 文件写入路径开始双写 `FileBlob + FileEntity.VERSION`:普通代理上传、直传完成、外部文件导入、分享导入,以及网盘复制复用 blob 时,都会给新 `StoredFile` 写入 `primaryEntity` 并创建 `StoredFileEntity(PRIMARY)` 关系。
|
||||
- 当前仍不切换读取路径:下载、ZIP、分享详情、回收站等旧业务继续依赖 `StoredFile.blob`,`primaryEntity` 只作为后续版本、缩略图、转码、存储策略迁移的兼容数据。
|
||||
- 为避免新关系表阻塞现有删除和测试清理,`StoredFileEntity -> StoredFile` 使用数据库级删除级联;`FileEntity.createdBy` 删除用户时置空,保留物理实体审计数据但不阻塞用户清理。
|
||||
|
||||
Reference in New Issue
Block a user