yoyuz/my_site
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(files): add storage policy skeleton

Browse Source
This commit is contained in:
yoyuzh
2026-04-08 15:37:43 +08:00
parent f582e600aa
commit 6da0d196ee
16 changed files with 507 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Controller.java
View File

@@ -85,6 +85,7 @@ public class UploadSessionV2Controller {
session.getFilename(),
session.getContentType(),
session.getSize(),
session.getStoragePolicyId(),
session.getStatus().name(),
session.getChunkSize(),
session.getChunkCount(),

1
backend/src/main/java/com/yoyuzh/api/v2/files/UploadSessionV2Response.java
View File

@@ -9,6 +9,7 @@ public record UploadSessionV2Response(
String filename,
String contentType,
long size,
Long storagePolicyId,
String status,
long chunkSize,
int chunkCount,

207
backend/src/main/java/com/yoyuzh/files/StoragePolicy.java Normal file
View File

@@ -0,0 +1,207 @@
package com.yoyuzh.files;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.Index;
import jakarta.persistence.PrePersist;
import jakarta.persistence.PreUpdate;
import jakarta.persistence.Table;
import java.time.LocalDateTime;
@Entity
@Table(name = "portal_storage_policy", indexes = {
@Index(name = "idx_storage_policy_enabled", columnList = "enabled"),
@Index(name = "idx_storage_policy_default", columnList = "default_policy")
})
public class StoragePolicy {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column(nullable = false, length = 128)
private String name;
@Enumerated(EnumType.STRING)
@Column(nullable = false, length = 32)
private StoragePolicyType type;
@Column(name = "bucket_name", length = 255)
private String bucketName;
@Column(length = 512)
private String endpoint;
@Column(length = 64)
private String region;
@Column(name = "private_bucket", nullable = false)
private boolean privateBucket;
@Column(length = 512)
private String prefix;
@Enumerated(EnumType.STRING)
@Column(name = "credential_mode", nullable = false, length = 32)
private StoragePolicyCredentialMode credentialMode;
@Column(name = "max_size_bytes", nullable = false)
private long maxSizeBytes;
@Column(name = "capabilities_json", columnDefinition = "TEXT")
private String capabilitiesJson;
@Column(nullable = false)
private boolean enabled;
@Column(name = "default_policy", nullable = false)
private boolean defaultPolicy;
@Column(name = "created_at", nullable = false)
private LocalDateTime createdAt;
@Column(name = "updated_at", nullable = false)
private LocalDateTime updatedAt;
@PrePersist
public void prePersist() {
LocalDateTime now = LocalDateTime.now();
if (createdAt == null) {
createdAt = now;
}
if (updatedAt == null) {
updatedAt = now;
}
}
@PreUpdate
public void preUpdate() {
updatedAt = LocalDateTime.now();
}
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public StoragePolicyType getType() {
return type;
}
public void setType(StoragePolicyType type) {
this.type = type;
}
public String getBucketName() {
return bucketName;
}
public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
public String getEndpoint() {
return endpoint;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public boolean isPrivateBucket() {
return privateBucket;
}
public void setPrivateBucket(boolean privateBucket) {
this.privateBucket = privateBucket;
}
public String getPrefix() {
return prefix;
}
public void setPrefix(String prefix) {
this.prefix = prefix;
}
public StoragePolicyCredentialMode getCredentialMode() {
return credentialMode;
}
public void setCredentialMode(StoragePolicyCredentialMode credentialMode) {
this.credentialMode = credentialMode;
}
public long getMaxSizeBytes() {
return maxSizeBytes;
}
public void setMaxSizeBytes(long maxSizeBytes) {
this.maxSizeBytes = maxSizeBytes;
}
public String getCapabilitiesJson() {
return capabilitiesJson;
}
public void setCapabilitiesJson(String capabilitiesJson) {
this.capabilitiesJson = capabilitiesJson;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public boolean isDefaultPolicy() {
return defaultPolicy;
}
public void setDefaultPolicy(boolean defaultPolicy) {
this.defaultPolicy = defaultPolicy;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public void setCreatedAt(LocalDateTime createdAt) {
this.createdAt = createdAt;
}
public LocalDateTime getUpdatedAt() {
return updatedAt;
}
public void setUpdatedAt(LocalDateTime updatedAt) {
this.updatedAt = updatedAt;
}
}

14
backend/src/main/java/com/yoyuzh/files/StoragePolicyCapabilities.java Normal file
View File

@@ -0,0 +1,14 @@
package com.yoyuzh.files;
public record StoragePolicyCapabilities(
boolean directUpload,
boolean multipartUpload,
boolean signedDownloadUrl,
boolean serverProxyDownload,
boolean thumbnailNative,
boolean friendlyDownloadName,
boolean requiresCors,
boolean supportsInternalEndpoint,
long maxObjectSize
) {
}

7
backend/src/main/java/com/yoyuzh/files/StoragePolicyCredentialMode.java Normal file
View File

@@ -0,0 +1,7 @@
package com.yoyuzh.files;
public enum StoragePolicyCredentialMode {
NONE,
STATIC,
DOGECLOUD_TEMP
}

10
backend/src/main/java/com/yoyuzh/files/StoragePolicyRepository.java Normal file
View File

@@ -0,0 +1,10 @@
package com.yoyuzh.files;
import org.springframework.data.jpa.repository.JpaRepository;
import java.util.Optional;
public interface StoragePolicyRepository extends JpaRepository<StoragePolicy, Long> {
Optional<StoragePolicy> findFirstByDefaultPolicyTrueOrderByIdAsc();
}

121
backend/src/main/java/com/yoyuzh/files/StoragePolicyService.java Normal file
View File

@@ -0,0 +1,121 @@
package com.yoyuzh.files;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.config.FileStorageProperties;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
@Service
@Order(-1)
@RequiredArgsConstructor
public class StoragePolicyService implements CommandLineRunner {
private final StoragePolicyRepository storagePolicyRepository;
private final FileStorageProperties properties;
private final ObjectMapper objectMapper = new ObjectMapper();
@Override
@Transactional
public void run(String... args) {
ensureDefaultPolicy();
}
@Transactional
public StoragePolicy ensureDefaultPolicy() {
return storagePolicyRepository.findFirstByDefaultPolicyTrueOrderByIdAsc()
.orElseGet(() -> storagePolicyRepository.save(createDefaultPolicy()));
}
public StoragePolicyCapabilities readCapabilities(StoragePolicy policy) {
try {
return objectMapper.readValue(policy.getCapabilitiesJson(), StoragePolicyCapabilities.class);
} catch (Exception ex) {
throw new IllegalStateException("Storage policy capabilities are invalid", ex);
}
}
private StoragePolicy createDefaultPolicy() {
if ("s3".equalsIgnoreCase(properties.getProvider())) {
return createDefaultS3Policy();
}
return createDefaultLocalPolicy();
}
private StoragePolicy createDefaultS3Policy() {
StoragePolicy policy = new StoragePolicy();
policy.setName("Default S3 Compatible Storage");
policy.setType(StoragePolicyType.S3_COMPATIBLE);
policy.setBucketName(extractScopeBucketName(properties.getS3().getScope()));
policy.setRegion(properties.getS3().getRegion());
policy.setPrivateBucket(true);
policy.setPrefix(extractScopePrefix(properties.getS3().getScope()));
policy.setCredentialMode(StoragePolicyCredentialMode.DOGECLOUD_TEMP);
policy.setMaxSizeBytes(properties.getMaxFileSize());
policy.setCapabilitiesJson(writeCapabilities(new StoragePolicyCapabilities(
true,
false,
true,
true,
false,
true,
true,
false,
properties.getMaxFileSize()
)));
policy.setEnabled(true);
policy.setDefaultPolicy(true);
return policy;
}
private StoragePolicy createDefaultLocalPolicy() {
StoragePolicy policy = new StoragePolicy();
policy.setName("Default Local Storage");
policy.setType(StoragePolicyType.LOCAL);
policy.setPrivateBucket(true);
policy.setPrefix(properties.getLocal().getRootDir());
policy.setCredentialMode(StoragePolicyCredentialMode.NONE);
policy.setMaxSizeBytes(properties.getMaxFileSize());
policy.setCapabilitiesJson(writeCapabilities(new StoragePolicyCapabilities(
false,
false,
false,
true,
false,
true,
false,
false,
properties.getMaxFileSize()
)));
policy.setEnabled(true);
policy.setDefaultPolicy(true);
return policy;
}
private String writeCapabilities(StoragePolicyCapabilities capabilities) {
try {
return objectMapper.writeValueAsString(capabilities);
} catch (Exception ex) {
throw new IllegalStateException("Storage policy capabilities cannot be serialized", ex);
}
}
private String extractScopeBucketName(String scope) {
if (!StringUtils.hasText(scope)) {
return null;
}
int separatorIndex = scope.indexOf(':');
return separatorIndex >= 0 ? scope.substring(0, separatorIndex) : scope;
}
private String extractScopePrefix(String scope) {
if (!StringUtils.hasText(scope)) {
return "";
}
int separatorIndex = scope.indexOf(':');
return separatorIndex >= 0 ? scope.substring(separatorIndex + 1) : "";
}
}

6
backend/src/main/java/com/yoyuzh/files/StoragePolicyType.java Normal file
View File

@@ -0,0 +1,6 @@
package com.yoyuzh.files;
public enum StoragePolicyType {
LOCAL,
S3_COMPATIBLE
}

11
backend/src/main/java/com/yoyuzh/files/UploadSession.java
View File

@@ -55,6 +55,9 @@ public class UploadSession {
@Column(name = "object_key", nullable = false, length = 512)
private String objectKey;
@Column(name = "storage_policy_id")
private Long storagePolicyId;
@Column(name = "chunk_size", nullable = false)
private Long chunkSize;
@@ -160,6 +163,14 @@ public class UploadSession {
this.objectKey = objectKey;
}
public Long getStoragePolicyId() {
return storagePolicyId;
}
public void setStoragePolicyId(Long storagePolicyId) {
this.storagePolicyId = storagePolicyId;
}
public Long getChunkSize() {
return chunkSize;
}

7
backend/src/main/java/com/yoyuzh/files/UploadSessionService.java
View File

@@ -37,6 +37,7 @@ public class UploadSessionService {
private final StoredFileRepository storedFileRepository;
private final FileService fileService;
private final FileContentStorage fileContentStorage;
private final StoragePolicyService storagePolicyService;
private final ObjectMapper objectMapper = new ObjectMapper();
private final long maxFileSize;
private final Clock clock;
@@ -46,20 +47,23 @@ public class UploadSessionService {
StoredFileRepository storedFileRepository,
FileService fileService,
FileContentStorage fileContentStorage,
StoragePolicyService storagePolicyService,
FileStorageProperties properties) {
this(uploadSessionRepository, storedFileRepository, fileService, fileContentStorage, properties, Clock.systemUTC());
this(uploadSessionRepository, storedFileRepository, fileService, fileContentStorage, storagePolicyService, properties, Clock.systemUTC());
}
UploadSessionService(UploadSessionRepository uploadSessionRepository,
StoredFileRepository storedFileRepository,
FileService fileService,
FileContentStorage fileContentStorage,
StoragePolicyService storagePolicyService,
FileStorageProperties properties,
Clock clock) {
this.uploadSessionRepository = uploadSessionRepository;
this.storedFileRepository = storedFileRepository;
this.fileService = fileService;
this.fileContentStorage = fileContentStorage;
this.storagePolicyService = storagePolicyService;
this.maxFileSize = properties.getMaxFileSize();
this.clock = clock;
}
@@ -78,6 +82,7 @@ public class UploadSessionService {
session.setContentType(command.contentType());
session.setSize(command.size());
session.setObjectKey(createBlobObjectKey());
session.setStoragePolicyId(storagePolicyService.ensureDefaultPolicy().getId());
session.setChunkSize(DEFAULT_CHUNK_SIZE);
session.setChunkCount(calculateChunkCount(command.size(), DEFAULT_CHUNK_SIZE));
session.setUploadedPartsJson("[]");

103
backend/src/test/java/com/yoyuzh/files/StoragePolicyServiceTest.java Normal file
View File

@@ -0,0 +1,103 @@
package com.yoyuzh.files;
import com.yoyuzh.config.FileStorageProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class StoragePolicyServiceTest {
@Mock
private StoragePolicyRepository storagePolicyRepository;
private FileStorageProperties properties;
private StoragePolicyService storagePolicyService;
@BeforeEach
void setUp() {
properties = new FileStorageProperties();
storagePolicyService = new StoragePolicyService(storagePolicyRepository, properties);
}
@Test
void shouldCreateDefaultS3CompatiblePolicyFromCurrentStorageConfig() {
properties.setProvider("s3");
properties.setMaxFileSize(5000L);
properties.getS3().setScope("media-bucket:portal-prefix");
properties.getS3().setRegion("automatic");
when(storagePolicyRepository.findFirstByDefaultPolicyTrueOrderByIdAsc()).thenReturn(Optional.empty());
when(storagePolicyRepository.save(any(StoragePolicy.class))).thenAnswer(invocation -> {
StoragePolicy policy = invocation.getArgument(0);
policy.setId(1L);
return policy;
});
StoragePolicy policy = storagePolicyService.ensureDefaultPolicy();
assertThat(policy.getName()).isEqualTo("Default S3 Compatible Storage");
assertThat(policy.getType()).isEqualTo(StoragePolicyType.S3_COMPATIBLE);
assertThat(policy.getCredentialMode()).isEqualTo(StoragePolicyCredentialMode.DOGECLOUD_TEMP);
assertThat(policy.getBucketName()).isEqualTo("media-bucket");
assertThat(policy.getPrefix()).isEqualTo("portal-prefix");
assertThat(policy.getRegion()).isEqualTo("automatic");
assertThat(policy.isDefaultPolicy()).isTrue();
assertThat(policy.isEnabled()).isTrue();
StoragePolicyCapabilities capabilities = storagePolicyService.readCapabilities(policy);
assertThat(capabilities.directUpload()).isTrue();
assertThat(capabilities.multipartUpload()).isFalse();
assertThat(capabilities.signedDownloadUrl()).isTrue();
assertThat(capabilities.serverProxyDownload()).isTrue();
assertThat(capabilities.requiresCors()).isTrue();
assertThat(capabilities.maxObjectSize()).isEqualTo(5000L);
}
@Test
void shouldCreateDefaultLocalPolicyFromCurrentStorageConfig() {
properties.setProvider("local");
properties.setMaxFileSize(2048L);
properties.getLocal().setRootDir("./storage");
when(storagePolicyRepository.findFirstByDefaultPolicyTrueOrderByIdAsc()).thenReturn(Optional.empty());
when(storagePolicyRepository.save(any(StoragePolicy.class))).thenAnswer(invocation -> invocation.getArgument(0));
StoragePolicy policy = storagePolicyService.ensureDefaultPolicy();
assertThat(policy.getName()).isEqualTo("Default Local Storage");
assertThat(policy.getType()).isEqualTo(StoragePolicyType.LOCAL);
assertThat(policy.getCredentialMode()).isEqualTo(StoragePolicyCredentialMode.NONE);
assertThat(policy.getPrefix()).isEqualTo("./storage");
StoragePolicyCapabilities capabilities = storagePolicyService.readCapabilities(policy);
assertThat(capabilities.directUpload()).isFalse();
assertThat(capabilities.multipartUpload()).isFalse();
assertThat(capabilities.signedDownloadUrl()).isFalse();
assertThat(capabilities.serverProxyDownload()).isTrue();
assertThat(capabilities.requiresCors()).isFalse();
assertThat(capabilities.maxObjectSize()).isEqualTo(2048L);
}
@Test
void shouldReuseExistingDefaultPolicy() {
StoragePolicy existingPolicy = new StoragePolicy();
existingPolicy.setId(7L);
existingPolicy.setDefaultPolicy(true);
existingPolicy.setEnabled(true);
when(storagePolicyRepository.findFirstByDefaultPolicyTrueOrderByIdAsc()).thenReturn(Optional.of(existingPolicy));
StoragePolicy policy = storagePolicyService.ensureDefaultPolicy();
assertThat(policy).isSameAs(existingPolicy);
verify(storagePolicyRepository, never()).save(any(StoragePolicy.class));
}
}

15
backend/src/test/java/com/yoyuzh/files/UploadSessionServiceTest.java
View File

@@ -37,6 +37,8 @@ class UploadSessionServiceTest {
private FileService fileService;
@Mock
private FileContentStorage fileContentStorage;
@Mock
private StoragePolicyService storagePolicyService;
private UploadSessionService uploadSessionService;
@@ -49,6 +51,7 @@ class UploadSessionServiceTest {
storedFileRepository,
fileService,
fileContentStorage,
storagePolicyService,
properties,
Clock.fixed(Instant.parse("2026-04-08T06:00:00Z"), ZoneOffset.UTC)
);
@@ -58,6 +61,7 @@ class UploadSessionServiceTest {
void shouldCreateUploadSessionWithoutChangingLegacyUploadPath() {
User user = createUser(7L);
when(storedFileRepository.existsByUserIdAndPathAndFilename(7L, "/docs", "movie.mp4")).thenReturn(false);
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy());
when(uploadSessionRepository.save(any(UploadSession.class))).thenAnswer(invocation -> {
UploadSession session = invocation.getArgument(0);
session.setId(100L);
@@ -72,6 +76,7 @@ class UploadSessionServiceTest {
assertThat(session.getSessionId()).isNotBlank();
assertThat(session.getObjectKey()).startsWith("blobs/");
assertThat(session.getStatus()).isEqualTo(UploadSessionStatus.CREATED);
assertThat(session.getStoragePolicyId()).isEqualTo(42L);
assertThat(session.getChunkSize()).isEqualTo(8L * 1024 * 1024);
assertThat(session.getChunkCount()).isEqualTo(3);
assertThat(session.getExpiresAt()).isEqualTo(LocalDateTime.of(2026, 4, 9, 6, 0));
@@ -214,6 +219,16 @@ class UploadSessionServiceTest {
return user;
}
private StoragePolicy createDefaultStoragePolicy() {
StoragePolicy policy = new StoragePolicy();
policy.setId(42L);
policy.setName("Default S3 Compatible Storage");
policy.setType(StoragePolicyType.S3_COMPATIBLE);
policy.setEnabled(true);
policy.setDefaultPolicy(true);
return policy;
}
private UploadSession createSession(User user) {
UploadSession session = new UploadSession();
session.setSessionId("session-1");