yoyuz/my_site
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(files): stamp entities with storage policy

Browse Source
This commit is contained in:
yoyuzh
2026-04-08 21:44:38 +08:00
parent 00b268c30f
commit 3e67760712
6 changed files with 54 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/com/yoyuzh/files/FileEntityBackfillService.java
View File

@@ -18,6 +18,7 @@ public class FileEntityBackfillService implements CommandLineRunner {
private final StoredFileRepository storedFileRepository;
private final FileEntityRepository fileEntityRepository;
private final StoredFileEntityRepository storedFileEntityRepository;
private final StoragePolicyService storagePolicyService;
@Override
@Transactional
@@ -51,6 +52,7 @@ public class FileEntityBackfillService implements CommandLineRunner {
fileEntity.setEntityType(FileEntityType.VERSION);
fileEntity.setReferenceCount(1);
fileEntity.setCreatedBy(storedFile.getUser());
fileEntity.setStoragePolicyId(storagePolicyService.ensureDefaultPolicy().getId());
return fileEntityRepository.save(fileEntity);
}

18
backend/src/main/java/com/yoyuzh/files/FileService.java
View File

@@ -56,6 +56,7 @@ public class FileService {
private final FileContentStorage fileContentStorage;
private final FileShareLinkRepository fileShareLinkRepository;
private final AdminMetricsService adminMetricsService;
private final StoragePolicyService storagePolicyService;
private final long maxFileSize;
private final String packageDownloadBaseUrl;
private final String packageDownloadSecret;
@@ -70,8 +71,9 @@ public class FileService {
FileContentStorage fileContentStorage,
FileShareLinkRepository fileShareLinkRepository,
AdminMetricsService adminMetricsService,
StoragePolicyService storagePolicyService,
FileStorageProperties properties) {
this(storedFileRepository, fileBlobRepository, fileEntityRepository, storedFileEntityRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC());
this(storedFileRepository, fileBlobRepository, fileEntityRepository, storedFileEntityRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, storagePolicyService, properties, Clock.systemUTC());
}
FileService(StoredFileRepository storedFileRepository,
@@ -81,6 +83,7 @@ public class FileService {
FileContentStorage fileContentStorage,
FileShareLinkRepository fileShareLinkRepository,
AdminMetricsService adminMetricsService,
StoragePolicyService storagePolicyService,
FileStorageProperties properties,
Clock clock) {
this.storedFileRepository = storedFileRepository;
@@ -90,6 +93,7 @@ public class FileService {
this.fileContentStorage = fileContentStorage;
this.fileShareLinkRepository = fileShareLinkRepository;
this.adminMetricsService = adminMetricsService;
this.storagePolicyService = storagePolicyService;
this.maxFileSize = properties.getMaxFileSize();
this.packageDownloadBaseUrl = StringUtils.hasText(properties.getS3().getPackageDownloadBaseUrl())
? properties.getS3().getPackageDownloadBaseUrl().trim()
@@ -107,7 +111,7 @@ public class FileService {
FileShareLinkRepository fileShareLinkRepository,
AdminMetricsService adminMetricsService,
FileStorageProperties properties) {
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC());
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, null, properties, Clock.systemUTC());
}
FileService(StoredFileRepository storedFileRepository,
@@ -117,7 +121,7 @@ public class FileService {
AdminMetricsService adminMetricsService,
FileStorageProperties properties,
Clock clock) {
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, clock);
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, null, properties, clock);
}
@Transactional
@@ -745,9 +749,17 @@ public class FileService {
entity.setEntityType(FileEntityType.VERSION);
entity.setReferenceCount(1);
entity.setCreatedBy(user);
entity.setStoragePolicyId(resolveDefaultStoragePolicyId());
return entity;
}
private Long resolveDefaultStoragePolicyId() {
if (storagePolicyService == null) {
return null;
}
return storagePolicyService.ensureDefaultPolicy().getId();
}
private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) {
if (storedFileEntityRepository == null) {
return;

19
backend/src/test/java/com/yoyuzh/files/FileEntityBackfillServiceTest.java
View File

@@ -26,6 +26,8 @@ class FileEntityBackfillServiceTest {
private FileEntityRepository fileEntityRepository;
@Mock
private StoredFileEntityRepository storedFileEntityRepository;
@Mock
private StoragePolicyService storagePolicyService;
private FileEntityBackfillService backfillService;
@@ -34,7 +36,8 @@ class FileEntityBackfillServiceTest {
backfillService = new FileEntityBackfillService(
storedFileRepository,
fileEntityRepository,
storedFileEntityRepository
storedFileEntityRepository,
storagePolicyService
);
}
@@ -50,6 +53,7 @@ class FileEntityBackfillServiceTest {
entity.setId(100L);
return entity;
});
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy());
backfillService.backfillPrimaryEntities();
@@ -57,6 +61,7 @@ class FileEntityBackfillServiceTest {
assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo("blobs/blob-20");
assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION);
assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1);
assertThat(storedFile.getPrimaryEntity().getStoragePolicyId()).isEqualTo(42L);
verify(fileEntityRepository).save(any(FileEntity.class));
verify(storedFileRepository).save(storedFile);
verify(storedFileEntityRepository).save(any(StoredFileEntity.class));
@@ -111,4 +116,16 @@ class FileEntityBackfillServiceTest {
blob.setCreatedAt(LocalDateTime.now());
return blob;
}
private StoragePolicy createDefaultStoragePolicy() {
StoragePolicy policy = new StoragePolicy();
policy.setId(42L);
policy.setName("Default Local Storage");
policy.setType(StoragePolicyType.LOCAL);
policy.setCredentialMode(StoragePolicyCredentialMode.NONE);
policy.setMaxSizeBytes(500L * 1024 * 1024);
policy.setEnabled(true);
policy.setDefaultPolicy(true);
return policy;
}
}

17
backend/src/test/java/com/yoyuzh/files/FileServiceTest.java
View File

@@ -63,6 +63,8 @@ class FileServiceTest {
private FileShareLinkRepository fileShareLinkRepository;
@Mock
private AdminMetricsService adminMetricsService;
@Mock
private StoragePolicyService storagePolicyService;
private FileService fileService;
@@ -150,6 +152,7 @@ class FileServiceTest {
fileContentStorage,
fileShareLinkRepository,
adminMetricsService,
storagePolicyService,
new FileStorageProperties()
);
User user = createUser(7L);
@@ -168,6 +171,7 @@ class FileServiceTest {
entity.setId(200L);
return entity;
});
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy());
when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> {
StoredFile file = invocation.getArgument(0);
file.setId(10L);
@@ -181,6 +185,7 @@ class FileServiceTest {
assertThat(entityCaptor.getValue().getObjectKey()).startsWith("blobs/");
assertThat(entityCaptor.getValue().getEntityType()).isEqualTo(FileEntityType.VERSION);
assertThat(entityCaptor.getValue().getCreatedBy()).isSameAs(user);
assertThat(entityCaptor.getValue().getStoragePolicyId()).isEqualTo(42L);
var relationCaptor = forClass(StoredFileEntity.class);
verify(storedFileEntityRepository).save(relationCaptor.capture());
@@ -828,4 +833,16 @@ class FileServiceTest {
directory.setBlob(null);
return directory;
}
private StoragePolicy createDefaultStoragePolicy() {
StoragePolicy policy = new StoragePolicy();
policy.setId(42L);
policy.setName("Default Local Storage");
policy.setType(StoragePolicyType.LOCAL);
policy.setCredentialMode(StoragePolicyCredentialMode.NONE);
policy.setMaxSizeBytes(500L * 1024 * 1024);
policy.setEnabled(true);
policy.setDefaultPolicy(true);
return policy;
}
}

1
docs/architecture.md
View File

@@ -451,3 +451,4 @@ Android 壳补充说明：
- 2026-04-08 阶段 3 第四小步补充：上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话，尝试删除 `objectKey` 对应的临时 blob，然后标记为 `EXPIRED`。已完成文件不参与清理，避免误删已经落库的生产对象。
- 2026-04-08 阶段 4 第一小步补充：后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略，并把当前 `FileStorageProperties` 映射为 `LOCAL` 或 `S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON；当前能力声明中 `multipartUpload=false`，用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID，但 `FileContentStorage` 仍保持单对象上传/校验抽象，旧 `/api/files/**` 生产路径不切换。
- 2026-04-08 `files/storage` 合并补充：S3 存储实现拆出多吉云临时密钥客户端与运行期会话提供器。`S3FileContentStorage` 现在通过 `S3SessionProvider.currentSession()` 获取当前 bucket、`S3Client` 和 `S3Presigner`，避免每次操作重复内联多吉云 token 解析逻辑；测试环境可直接注入 mock S3 client/presigner。该改动没有引入 multipart，仍是单对象 PUT/HEAD/GET/COPY/DELETE 路径。
- 2026-04-08 阶段 4 第二小步补充：`FileService` 在创建新的 `FileEntity.VERSION` 时会通过 `StoragePolicyService.ensureDefaultPolicy()` 写入默认 `storagePolicyId`；`FileEntityBackfillService` 对历史 `FileBlob` 回填新实体时也写入同一默认策略。复用已有实体时保持原策略字段不变，只增加引用计数，避免在兼容迁移阶段覆盖历史数据。

1
memory.md
View File

@@ -166,3 +166,4 @@
- 2026-04-08 multipart 评估结论：暂不把 v2 上传会话直接接入真实对象存储分片写入/合并。当前 `FileContentStorage` 仍是单对象上传/校验抽象，缺少 multipart uploadId、part URL 预签名、complete/abort 语义；立即接入会把上传会话写死在当前多吉云 S3 配置上，并让过期清理误以为 `deleteBlob` 能释放未完成分片。下一步先做阶段 4 存储策略与能力声明骨架，再按 `multipartUpload` 能力接 S3 multipart。
- 2026-04-08 阶段 4 第一小步：新增 `StoragePolicy`、`StoragePolicyType`、`StoragePolicyCredentialMode`、`StoragePolicyCapabilities` 与 `StoragePolicyService`，启动时把当前 `app.storage.provider` 映射成一条默认策略；本地策略声明 `serverProxyDownload=true`、`multipartUpload=false`，多吉云/S3 兼容策略声明 `directUpload=true`、`signedDownloadUrl=true`、`requiresCors=true`、`multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`，但旧上传下载路径和前端上传队列仍未切换。
- 2026-04-08 合并 `files/storage` 补提交后修复：`S3FileContentStorage` 改为复用 `DogeCloudS3SessionProvider` / `DogeCloudTmpTokenClient` 获取并缓存运行期 `S3Client` 与 `S3Presigner`，保留生产构造器 `S3FileContentStorage(FileStorageProperties)`，同时提供测试用注入构造器；S3 直传、签名下载、上传校验、读旧对象键 fallback、rename/move/copy、离线快传对象读写继续通过 `FileContentStorage` 统一抽象。
- 2026-04-08 阶段 4 第二小步：新写入和回填生成的 `FileEntity.VERSION` 会记录默认 `StoragePolicy.id` 到 `storagePolicyId`，让物理实体可以追踪归属存储策略；复用已有 `FileEntity` 时只增加引用计数，不覆盖历史实体策略字段。旧 `/api/files/**` 读取路径仍继续依赖 `StoredFile.blob`。