feat(files): stamp entities with storage policy

This commit is contained in:
yoyuzh
2026-04-08 21:44:38 +08:00
parent 00b268c30f
commit 3e67760712
6 changed files with 54 additions and 4 deletions

View File

@@ -18,6 +18,7 @@ public class FileEntityBackfillService implements CommandLineRunner {
private final StoredFileRepository storedFileRepository; private final StoredFileRepository storedFileRepository;
private final FileEntityRepository fileEntityRepository; private final FileEntityRepository fileEntityRepository;
private final StoredFileEntityRepository storedFileEntityRepository; private final StoredFileEntityRepository storedFileEntityRepository;
private final StoragePolicyService storagePolicyService;
@Override @Override
@Transactional @Transactional
@@ -51,6 +52,7 @@ public class FileEntityBackfillService implements CommandLineRunner {
fileEntity.setEntityType(FileEntityType.VERSION); fileEntity.setEntityType(FileEntityType.VERSION);
fileEntity.setReferenceCount(1); fileEntity.setReferenceCount(1);
fileEntity.setCreatedBy(storedFile.getUser()); fileEntity.setCreatedBy(storedFile.getUser());
fileEntity.setStoragePolicyId(storagePolicyService.ensureDefaultPolicy().getId());
return fileEntityRepository.save(fileEntity); return fileEntityRepository.save(fileEntity);
} }

View File

@@ -56,6 +56,7 @@ public class FileService {
private final FileContentStorage fileContentStorage; private final FileContentStorage fileContentStorage;
private final FileShareLinkRepository fileShareLinkRepository; private final FileShareLinkRepository fileShareLinkRepository;
private final AdminMetricsService adminMetricsService; private final AdminMetricsService adminMetricsService;
private final StoragePolicyService storagePolicyService;
private final long maxFileSize; private final long maxFileSize;
private final String packageDownloadBaseUrl; private final String packageDownloadBaseUrl;
private final String packageDownloadSecret; private final String packageDownloadSecret;
@@ -70,8 +71,9 @@ public class FileService {
FileContentStorage fileContentStorage, FileContentStorage fileContentStorage,
FileShareLinkRepository fileShareLinkRepository, FileShareLinkRepository fileShareLinkRepository,
AdminMetricsService adminMetricsService, AdminMetricsService adminMetricsService,
StoragePolicyService storagePolicyService,
FileStorageProperties properties) { FileStorageProperties properties) {
this(storedFileRepository, fileBlobRepository, fileEntityRepository, storedFileEntityRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC()); this(storedFileRepository, fileBlobRepository, fileEntityRepository, storedFileEntityRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, storagePolicyService, properties, Clock.systemUTC());
} }
FileService(StoredFileRepository storedFileRepository, FileService(StoredFileRepository storedFileRepository,
@@ -81,6 +83,7 @@ public class FileService {
FileContentStorage fileContentStorage, FileContentStorage fileContentStorage,
FileShareLinkRepository fileShareLinkRepository, FileShareLinkRepository fileShareLinkRepository,
AdminMetricsService adminMetricsService, AdminMetricsService adminMetricsService,
StoragePolicyService storagePolicyService,
FileStorageProperties properties, FileStorageProperties properties,
Clock clock) { Clock clock) {
this.storedFileRepository = storedFileRepository; this.storedFileRepository = storedFileRepository;
@@ -90,6 +93,7 @@ public class FileService {
this.fileContentStorage = fileContentStorage; this.fileContentStorage = fileContentStorage;
this.fileShareLinkRepository = fileShareLinkRepository; this.fileShareLinkRepository = fileShareLinkRepository;
this.adminMetricsService = adminMetricsService; this.adminMetricsService = adminMetricsService;
this.storagePolicyService = storagePolicyService;
this.maxFileSize = properties.getMaxFileSize(); this.maxFileSize = properties.getMaxFileSize();
this.packageDownloadBaseUrl = StringUtils.hasText(properties.getS3().getPackageDownloadBaseUrl()) this.packageDownloadBaseUrl = StringUtils.hasText(properties.getS3().getPackageDownloadBaseUrl())
? properties.getS3().getPackageDownloadBaseUrl().trim() ? properties.getS3().getPackageDownloadBaseUrl().trim()
@@ -107,7 +111,7 @@ public class FileService {
FileShareLinkRepository fileShareLinkRepository, FileShareLinkRepository fileShareLinkRepository,
AdminMetricsService adminMetricsService, AdminMetricsService adminMetricsService,
FileStorageProperties properties) { FileStorageProperties properties) {
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC()); this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, null, properties, Clock.systemUTC());
} }
FileService(StoredFileRepository storedFileRepository, FileService(StoredFileRepository storedFileRepository,
@@ -117,7 +121,7 @@ public class FileService {
AdminMetricsService adminMetricsService, AdminMetricsService adminMetricsService,
FileStorageProperties properties, FileStorageProperties properties,
Clock clock) { Clock clock) {
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, clock); this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, null, properties, clock);
} }
@Transactional @Transactional
@@ -745,9 +749,17 @@ public class FileService {
entity.setEntityType(FileEntityType.VERSION); entity.setEntityType(FileEntityType.VERSION);
entity.setReferenceCount(1); entity.setReferenceCount(1);
entity.setCreatedBy(user); entity.setCreatedBy(user);
entity.setStoragePolicyId(resolveDefaultStoragePolicyId());
return entity; return entity;
} }
private Long resolveDefaultStoragePolicyId() {
if (storagePolicyService == null) {
return null;
}
return storagePolicyService.ensureDefaultPolicy().getId();
}
private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) { private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) {
if (storedFileEntityRepository == null) { if (storedFileEntityRepository == null) {
return; return;

View File

@@ -26,6 +26,8 @@ class FileEntityBackfillServiceTest {
private FileEntityRepository fileEntityRepository; private FileEntityRepository fileEntityRepository;
@Mock @Mock
private StoredFileEntityRepository storedFileEntityRepository; private StoredFileEntityRepository storedFileEntityRepository;
@Mock
private StoragePolicyService storagePolicyService;
private FileEntityBackfillService backfillService; private FileEntityBackfillService backfillService;
@@ -34,7 +36,8 @@ class FileEntityBackfillServiceTest {
backfillService = new FileEntityBackfillService( backfillService = new FileEntityBackfillService(
storedFileRepository, storedFileRepository,
fileEntityRepository, fileEntityRepository,
storedFileEntityRepository storedFileEntityRepository,
storagePolicyService
); );
} }
@@ -50,6 +53,7 @@ class FileEntityBackfillServiceTest {
entity.setId(100L); entity.setId(100L);
return entity; return entity;
}); });
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy());
backfillService.backfillPrimaryEntities(); backfillService.backfillPrimaryEntities();
@@ -57,6 +61,7 @@ class FileEntityBackfillServiceTest {
assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo("blobs/blob-20"); assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo("blobs/blob-20");
assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION); assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION);
assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1); assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1);
assertThat(storedFile.getPrimaryEntity().getStoragePolicyId()).isEqualTo(42L);
verify(fileEntityRepository).save(any(FileEntity.class)); verify(fileEntityRepository).save(any(FileEntity.class));
verify(storedFileRepository).save(storedFile); verify(storedFileRepository).save(storedFile);
verify(storedFileEntityRepository).save(any(StoredFileEntity.class)); verify(storedFileEntityRepository).save(any(StoredFileEntity.class));
@@ -111,4 +116,16 @@ class FileEntityBackfillServiceTest {
blob.setCreatedAt(LocalDateTime.now()); blob.setCreatedAt(LocalDateTime.now());
return blob; return blob;
} }
private StoragePolicy createDefaultStoragePolicy() {
StoragePolicy policy = new StoragePolicy();
policy.setId(42L);
policy.setName("Default Local Storage");
policy.setType(StoragePolicyType.LOCAL);
policy.setCredentialMode(StoragePolicyCredentialMode.NONE);
policy.setMaxSizeBytes(500L * 1024 * 1024);
policy.setEnabled(true);
policy.setDefaultPolicy(true);
return policy;
}
} }

View File

@@ -63,6 +63,8 @@ class FileServiceTest {
private FileShareLinkRepository fileShareLinkRepository; private FileShareLinkRepository fileShareLinkRepository;
@Mock @Mock
private AdminMetricsService adminMetricsService; private AdminMetricsService adminMetricsService;
@Mock
private StoragePolicyService storagePolicyService;
private FileService fileService; private FileService fileService;
@@ -150,6 +152,7 @@ class FileServiceTest {
fileContentStorage, fileContentStorage,
fileShareLinkRepository, fileShareLinkRepository,
adminMetricsService, adminMetricsService,
storagePolicyService,
new FileStorageProperties() new FileStorageProperties()
); );
User user = createUser(7L); User user = createUser(7L);
@@ -168,6 +171,7 @@ class FileServiceTest {
entity.setId(200L); entity.setId(200L);
return entity; return entity;
}); });
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy());
when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> { when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> {
StoredFile file = invocation.getArgument(0); StoredFile file = invocation.getArgument(0);
file.setId(10L); file.setId(10L);
@@ -181,6 +185,7 @@ class FileServiceTest {
assertThat(entityCaptor.getValue().getObjectKey()).startsWith("blobs/"); assertThat(entityCaptor.getValue().getObjectKey()).startsWith("blobs/");
assertThat(entityCaptor.getValue().getEntityType()).isEqualTo(FileEntityType.VERSION); assertThat(entityCaptor.getValue().getEntityType()).isEqualTo(FileEntityType.VERSION);
assertThat(entityCaptor.getValue().getCreatedBy()).isSameAs(user); assertThat(entityCaptor.getValue().getCreatedBy()).isSameAs(user);
assertThat(entityCaptor.getValue().getStoragePolicyId()).isEqualTo(42L);
var relationCaptor = forClass(StoredFileEntity.class); var relationCaptor = forClass(StoredFileEntity.class);
verify(storedFileEntityRepository).save(relationCaptor.capture()); verify(storedFileEntityRepository).save(relationCaptor.capture());
@@ -828,4 +833,16 @@ class FileServiceTest {
directory.setBlob(null); directory.setBlob(null);
return directory; return directory;
} }
private StoragePolicy createDefaultStoragePolicy() {
StoragePolicy policy = new StoragePolicy();
policy.setId(42L);
policy.setName("Default Local Storage");
policy.setType(StoragePolicyType.LOCAL);
policy.setCredentialMode(StoragePolicyCredentialMode.NONE);
policy.setMaxSizeBytes(500L * 1024 * 1024);
policy.setEnabled(true);
policy.setDefaultPolicy(true);
return policy;
}
} }

View File

@@ -451,3 +451,4 @@ Android 壳补充说明:
- 2026-04-08 阶段 3 第四小步补充:上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话,尝试删除 `objectKey` 对应的临时 blob然后标记为 `EXPIRED`。已完成文件不参与清理,避免误删已经落库的生产对象。 - 2026-04-08 阶段 3 第四小步补充:上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话,尝试删除 `objectKey` 对应的临时 blob然后标记为 `EXPIRED`。已完成文件不参与清理,避免误删已经落库的生产对象。
- 2026-04-08 阶段 4 第一小步补充:后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略,并把当前 `FileStorageProperties` 映射为 `LOCAL``S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON当前能力声明中 `multipartUpload=false`,用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID`FileContentStorage` 仍保持单对象上传/校验抽象,旧 `/api/files/**` 生产路径不切换。 - 2026-04-08 阶段 4 第一小步补充:后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略,并把当前 `FileStorageProperties` 映射为 `LOCAL``S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON当前能力声明中 `multipartUpload=false`,用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID`FileContentStorage` 仍保持单对象上传/校验抽象,旧 `/api/files/**` 生产路径不切换。
- 2026-04-08 `files/storage` 合并补充S3 存储实现拆出多吉云临时密钥客户端与运行期会话提供器。`S3FileContentStorage` 现在通过 `S3SessionProvider.currentSession()` 获取当前 bucket、`S3Client``S3Presigner`,避免每次操作重复内联多吉云 token 解析逻辑;测试环境可直接注入 mock S3 client/presigner。该改动没有引入 multipart仍是单对象 PUT/HEAD/GET/COPY/DELETE 路径。 - 2026-04-08 `files/storage` 合并补充S3 存储实现拆出多吉云临时密钥客户端与运行期会话提供器。`S3FileContentStorage` 现在通过 `S3SessionProvider.currentSession()` 获取当前 bucket、`S3Client``S3Presigner`,避免每次操作重复内联多吉云 token 解析逻辑;测试环境可直接注入 mock S3 client/presigner。该改动没有引入 multipart仍是单对象 PUT/HEAD/GET/COPY/DELETE 路径。
- 2026-04-08 阶段 4 第二小步补充:`FileService` 在创建新的 `FileEntity.VERSION` 时会通过 `StoragePolicyService.ensureDefaultPolicy()` 写入默认 `storagePolicyId``FileEntityBackfillService` 对历史 `FileBlob` 回填新实体时也写入同一默认策略。复用已有实体时保持原策略字段不变,只增加引用计数,避免在兼容迁移阶段覆盖历史数据。

View File

@@ -166,3 +166,4 @@
- 2026-04-08 multipart 评估结论:暂不把 v2 上传会话直接接入真实对象存储分片写入/合并。当前 `FileContentStorage` 仍是单对象上传/校验抽象,缺少 multipart uploadId、part URL 预签名、complete/abort 语义;立即接入会把上传会话写死在当前多吉云 S3 配置上,并让过期清理误以为 `deleteBlob` 能释放未完成分片。下一步先做阶段 4 存储策略与能力声明骨架,再按 `multipartUpload` 能力接 S3 multipart。 - 2026-04-08 multipart 评估结论:暂不把 v2 上传会话直接接入真实对象存储分片写入/合并。当前 `FileContentStorage` 仍是单对象上传/校验抽象,缺少 multipart uploadId、part URL 预签名、complete/abort 语义;立即接入会把上传会话写死在当前多吉云 S3 配置上,并让过期清理误以为 `deleteBlob` 能释放未完成分片。下一步先做阶段 4 存储策略与能力声明骨架,再按 `multipartUpload` 能力接 S3 multipart。
- 2026-04-08 阶段 4 第一小步:新增 `StoragePolicy``StoragePolicyType``StoragePolicyCredentialMode``StoragePolicyCapabilities``StoragePolicyService`,启动时把当前 `app.storage.provider` 映射成一条默认策略;本地策略声明 `serverProxyDownload=true``multipartUpload=false`,多吉云/S3 兼容策略声明 `directUpload=true``signedDownloadUrl=true``requiresCors=true``multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`,但旧上传下载路径和前端上传队列仍未切换。 - 2026-04-08 阶段 4 第一小步:新增 `StoragePolicy``StoragePolicyType``StoragePolicyCredentialMode``StoragePolicyCapabilities``StoragePolicyService`,启动时把当前 `app.storage.provider` 映射成一条默认策略;本地策略声明 `serverProxyDownload=true``multipartUpload=false`,多吉云/S3 兼容策略声明 `directUpload=true``signedDownloadUrl=true``requiresCors=true``multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`,但旧上传下载路径和前端上传队列仍未切换。
- 2026-04-08 合并 `files/storage` 补提交后修复:`S3FileContentStorage` 改为复用 `DogeCloudS3SessionProvider` / `DogeCloudTmpTokenClient` 获取并缓存运行期 `S3Client``S3Presigner`,保留生产构造器 `S3FileContentStorage(FileStorageProperties)`同时提供测试用注入构造器S3 直传、签名下载、上传校验、读旧对象键 fallback、rename/move/copy、离线快传对象读写继续通过 `FileContentStorage` 统一抽象。 - 2026-04-08 合并 `files/storage` 补提交后修复:`S3FileContentStorage` 改为复用 `DogeCloudS3SessionProvider` / `DogeCloudTmpTokenClient` 获取并缓存运行期 `S3Client``S3Presigner`,保留生产构造器 `S3FileContentStorage(FileStorageProperties)`同时提供测试用注入构造器S3 直传、签名下载、上传校验、读旧对象键 fallback、rename/move/copy、离线快传对象读写继续通过 `FileContentStorage` 统一抽象。
- 2026-04-08 阶段 4 第二小步:新写入和回填生成的 `FileEntity.VERSION` 会记录默认 `StoragePolicy.id``storagePolicyId`,让物理实体可以追踪归属存储策略;复用已有 `FileEntity` 时只增加引用计数,不覆盖历史实体策略字段。旧 `/api/files/**` 读取路径仍继续依赖 `StoredFile.blob`