feat(files): stamp entities with storage policy
This commit is contained in:
@@ -18,6 +18,7 @@ public class FileEntityBackfillService implements CommandLineRunner {
|
|||||||
private final StoredFileRepository storedFileRepository;
|
private final StoredFileRepository storedFileRepository;
|
||||||
private final FileEntityRepository fileEntityRepository;
|
private final FileEntityRepository fileEntityRepository;
|
||||||
private final StoredFileEntityRepository storedFileEntityRepository;
|
private final StoredFileEntityRepository storedFileEntityRepository;
|
||||||
|
private final StoragePolicyService storagePolicyService;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@Transactional
|
@Transactional
|
||||||
@@ -51,6 +52,7 @@ public class FileEntityBackfillService implements CommandLineRunner {
|
|||||||
fileEntity.setEntityType(FileEntityType.VERSION);
|
fileEntity.setEntityType(FileEntityType.VERSION);
|
||||||
fileEntity.setReferenceCount(1);
|
fileEntity.setReferenceCount(1);
|
||||||
fileEntity.setCreatedBy(storedFile.getUser());
|
fileEntity.setCreatedBy(storedFile.getUser());
|
||||||
|
fileEntity.setStoragePolicyId(storagePolicyService.ensureDefaultPolicy().getId());
|
||||||
return fileEntityRepository.save(fileEntity);
|
return fileEntityRepository.save(fileEntity);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ public class FileService {
|
|||||||
private final FileContentStorage fileContentStorage;
|
private final FileContentStorage fileContentStorage;
|
||||||
private final FileShareLinkRepository fileShareLinkRepository;
|
private final FileShareLinkRepository fileShareLinkRepository;
|
||||||
private final AdminMetricsService adminMetricsService;
|
private final AdminMetricsService adminMetricsService;
|
||||||
|
private final StoragePolicyService storagePolicyService;
|
||||||
private final long maxFileSize;
|
private final long maxFileSize;
|
||||||
private final String packageDownloadBaseUrl;
|
private final String packageDownloadBaseUrl;
|
||||||
private final String packageDownloadSecret;
|
private final String packageDownloadSecret;
|
||||||
@@ -70,8 +71,9 @@ public class FileService {
|
|||||||
FileContentStorage fileContentStorage,
|
FileContentStorage fileContentStorage,
|
||||||
FileShareLinkRepository fileShareLinkRepository,
|
FileShareLinkRepository fileShareLinkRepository,
|
||||||
AdminMetricsService adminMetricsService,
|
AdminMetricsService adminMetricsService,
|
||||||
|
StoragePolicyService storagePolicyService,
|
||||||
FileStorageProperties properties) {
|
FileStorageProperties properties) {
|
||||||
this(storedFileRepository, fileBlobRepository, fileEntityRepository, storedFileEntityRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC());
|
this(storedFileRepository, fileBlobRepository, fileEntityRepository, storedFileEntityRepository, fileContentStorage, fileShareLinkRepository, adminMetricsService, storagePolicyService, properties, Clock.systemUTC());
|
||||||
}
|
}
|
||||||
|
|
||||||
FileService(StoredFileRepository storedFileRepository,
|
FileService(StoredFileRepository storedFileRepository,
|
||||||
@@ -81,6 +83,7 @@ public class FileService {
|
|||||||
FileContentStorage fileContentStorage,
|
FileContentStorage fileContentStorage,
|
||||||
FileShareLinkRepository fileShareLinkRepository,
|
FileShareLinkRepository fileShareLinkRepository,
|
||||||
AdminMetricsService adminMetricsService,
|
AdminMetricsService adminMetricsService,
|
||||||
|
StoragePolicyService storagePolicyService,
|
||||||
FileStorageProperties properties,
|
FileStorageProperties properties,
|
||||||
Clock clock) {
|
Clock clock) {
|
||||||
this.storedFileRepository = storedFileRepository;
|
this.storedFileRepository = storedFileRepository;
|
||||||
@@ -90,6 +93,7 @@ public class FileService {
|
|||||||
this.fileContentStorage = fileContentStorage;
|
this.fileContentStorage = fileContentStorage;
|
||||||
this.fileShareLinkRepository = fileShareLinkRepository;
|
this.fileShareLinkRepository = fileShareLinkRepository;
|
||||||
this.adminMetricsService = adminMetricsService;
|
this.adminMetricsService = adminMetricsService;
|
||||||
|
this.storagePolicyService = storagePolicyService;
|
||||||
this.maxFileSize = properties.getMaxFileSize();
|
this.maxFileSize = properties.getMaxFileSize();
|
||||||
this.packageDownloadBaseUrl = StringUtils.hasText(properties.getS3().getPackageDownloadBaseUrl())
|
this.packageDownloadBaseUrl = StringUtils.hasText(properties.getS3().getPackageDownloadBaseUrl())
|
||||||
? properties.getS3().getPackageDownloadBaseUrl().trim()
|
? properties.getS3().getPackageDownloadBaseUrl().trim()
|
||||||
@@ -107,7 +111,7 @@ public class FileService {
|
|||||||
FileShareLinkRepository fileShareLinkRepository,
|
FileShareLinkRepository fileShareLinkRepository,
|
||||||
AdminMetricsService adminMetricsService,
|
AdminMetricsService adminMetricsService,
|
||||||
FileStorageProperties properties) {
|
FileStorageProperties properties) {
|
||||||
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, Clock.systemUTC());
|
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, null, properties, Clock.systemUTC());
|
||||||
}
|
}
|
||||||
|
|
||||||
FileService(StoredFileRepository storedFileRepository,
|
FileService(StoredFileRepository storedFileRepository,
|
||||||
@@ -117,7 +121,7 @@ public class FileService {
|
|||||||
AdminMetricsService adminMetricsService,
|
AdminMetricsService adminMetricsService,
|
||||||
FileStorageProperties properties,
|
FileStorageProperties properties,
|
||||||
Clock clock) {
|
Clock clock) {
|
||||||
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, properties, clock);
|
this(storedFileRepository, fileBlobRepository, null, null, fileContentStorage, fileShareLinkRepository, adminMetricsService, null, properties, clock);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
@@ -745,9 +749,17 @@ public class FileService {
|
|||||||
entity.setEntityType(FileEntityType.VERSION);
|
entity.setEntityType(FileEntityType.VERSION);
|
||||||
entity.setReferenceCount(1);
|
entity.setReferenceCount(1);
|
||||||
entity.setCreatedBy(user);
|
entity.setCreatedBy(user);
|
||||||
|
entity.setStoragePolicyId(resolveDefaultStoragePolicyId());
|
||||||
return entity;
|
return entity;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Long resolveDefaultStoragePolicyId() {
|
||||||
|
if (storagePolicyService == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return storagePolicyService.ensureDefaultPolicy().getId();
|
||||||
|
}
|
||||||
|
|
||||||
private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) {
|
private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) {
|
||||||
if (storedFileEntityRepository == null) {
|
if (storedFileEntityRepository == null) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ class FileEntityBackfillServiceTest {
|
|||||||
private FileEntityRepository fileEntityRepository;
|
private FileEntityRepository fileEntityRepository;
|
||||||
@Mock
|
@Mock
|
||||||
private StoredFileEntityRepository storedFileEntityRepository;
|
private StoredFileEntityRepository storedFileEntityRepository;
|
||||||
|
@Mock
|
||||||
|
private StoragePolicyService storagePolicyService;
|
||||||
|
|
||||||
private FileEntityBackfillService backfillService;
|
private FileEntityBackfillService backfillService;
|
||||||
|
|
||||||
@@ -34,7 +36,8 @@ class FileEntityBackfillServiceTest {
|
|||||||
backfillService = new FileEntityBackfillService(
|
backfillService = new FileEntityBackfillService(
|
||||||
storedFileRepository,
|
storedFileRepository,
|
||||||
fileEntityRepository,
|
fileEntityRepository,
|
||||||
storedFileEntityRepository
|
storedFileEntityRepository,
|
||||||
|
storagePolicyService
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -50,6 +53,7 @@ class FileEntityBackfillServiceTest {
|
|||||||
entity.setId(100L);
|
entity.setId(100L);
|
||||||
return entity;
|
return entity;
|
||||||
});
|
});
|
||||||
|
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy());
|
||||||
|
|
||||||
backfillService.backfillPrimaryEntities();
|
backfillService.backfillPrimaryEntities();
|
||||||
|
|
||||||
@@ -57,6 +61,7 @@ class FileEntityBackfillServiceTest {
|
|||||||
assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo("blobs/blob-20");
|
assertThat(storedFile.getPrimaryEntity().getObjectKey()).isEqualTo("blobs/blob-20");
|
||||||
assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION);
|
assertThat(storedFile.getPrimaryEntity().getEntityType()).isEqualTo(FileEntityType.VERSION);
|
||||||
assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1);
|
assertThat(storedFile.getPrimaryEntity().getReferenceCount()).isEqualTo(1);
|
||||||
|
assertThat(storedFile.getPrimaryEntity().getStoragePolicyId()).isEqualTo(42L);
|
||||||
verify(fileEntityRepository).save(any(FileEntity.class));
|
verify(fileEntityRepository).save(any(FileEntity.class));
|
||||||
verify(storedFileRepository).save(storedFile);
|
verify(storedFileRepository).save(storedFile);
|
||||||
verify(storedFileEntityRepository).save(any(StoredFileEntity.class));
|
verify(storedFileEntityRepository).save(any(StoredFileEntity.class));
|
||||||
@@ -111,4 +116,16 @@ class FileEntityBackfillServiceTest {
|
|||||||
blob.setCreatedAt(LocalDateTime.now());
|
blob.setCreatedAt(LocalDateTime.now());
|
||||||
return blob;
|
return blob;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private StoragePolicy createDefaultStoragePolicy() {
|
||||||
|
StoragePolicy policy = new StoragePolicy();
|
||||||
|
policy.setId(42L);
|
||||||
|
policy.setName("Default Local Storage");
|
||||||
|
policy.setType(StoragePolicyType.LOCAL);
|
||||||
|
policy.setCredentialMode(StoragePolicyCredentialMode.NONE);
|
||||||
|
policy.setMaxSizeBytes(500L * 1024 * 1024);
|
||||||
|
policy.setEnabled(true);
|
||||||
|
policy.setDefaultPolicy(true);
|
||||||
|
return policy;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,6 +63,8 @@ class FileServiceTest {
|
|||||||
private FileShareLinkRepository fileShareLinkRepository;
|
private FileShareLinkRepository fileShareLinkRepository;
|
||||||
@Mock
|
@Mock
|
||||||
private AdminMetricsService adminMetricsService;
|
private AdminMetricsService adminMetricsService;
|
||||||
|
@Mock
|
||||||
|
private StoragePolicyService storagePolicyService;
|
||||||
|
|
||||||
private FileService fileService;
|
private FileService fileService;
|
||||||
|
|
||||||
@@ -150,6 +152,7 @@ class FileServiceTest {
|
|||||||
fileContentStorage,
|
fileContentStorage,
|
||||||
fileShareLinkRepository,
|
fileShareLinkRepository,
|
||||||
adminMetricsService,
|
adminMetricsService,
|
||||||
|
storagePolicyService,
|
||||||
new FileStorageProperties()
|
new FileStorageProperties()
|
||||||
);
|
);
|
||||||
User user = createUser(7L);
|
User user = createUser(7L);
|
||||||
@@ -168,6 +171,7 @@ class FileServiceTest {
|
|||||||
entity.setId(200L);
|
entity.setId(200L);
|
||||||
return entity;
|
return entity;
|
||||||
});
|
});
|
||||||
|
when(storagePolicyService.ensureDefaultPolicy()).thenReturn(createDefaultStoragePolicy());
|
||||||
when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> {
|
when(storedFileRepository.save(any(StoredFile.class))).thenAnswer(invocation -> {
|
||||||
StoredFile file = invocation.getArgument(0);
|
StoredFile file = invocation.getArgument(0);
|
||||||
file.setId(10L);
|
file.setId(10L);
|
||||||
@@ -181,6 +185,7 @@ class FileServiceTest {
|
|||||||
assertThat(entityCaptor.getValue().getObjectKey()).startsWith("blobs/");
|
assertThat(entityCaptor.getValue().getObjectKey()).startsWith("blobs/");
|
||||||
assertThat(entityCaptor.getValue().getEntityType()).isEqualTo(FileEntityType.VERSION);
|
assertThat(entityCaptor.getValue().getEntityType()).isEqualTo(FileEntityType.VERSION);
|
||||||
assertThat(entityCaptor.getValue().getCreatedBy()).isSameAs(user);
|
assertThat(entityCaptor.getValue().getCreatedBy()).isSameAs(user);
|
||||||
|
assertThat(entityCaptor.getValue().getStoragePolicyId()).isEqualTo(42L);
|
||||||
|
|
||||||
var relationCaptor = forClass(StoredFileEntity.class);
|
var relationCaptor = forClass(StoredFileEntity.class);
|
||||||
verify(storedFileEntityRepository).save(relationCaptor.capture());
|
verify(storedFileEntityRepository).save(relationCaptor.capture());
|
||||||
@@ -828,4 +833,16 @@ class FileServiceTest {
|
|||||||
directory.setBlob(null);
|
directory.setBlob(null);
|
||||||
return directory;
|
return directory;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private StoragePolicy createDefaultStoragePolicy() {
|
||||||
|
StoragePolicy policy = new StoragePolicy();
|
||||||
|
policy.setId(42L);
|
||||||
|
policy.setName("Default Local Storage");
|
||||||
|
policy.setType(StoragePolicyType.LOCAL);
|
||||||
|
policy.setCredentialMode(StoragePolicyCredentialMode.NONE);
|
||||||
|
policy.setMaxSizeBytes(500L * 1024 * 1024);
|
||||||
|
policy.setEnabled(true);
|
||||||
|
policy.setDefaultPolicy(true);
|
||||||
|
return policy;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -451,3 +451,4 @@ Android 壳补充说明:
|
|||||||
- 2026-04-08 阶段 3 第四小步补充:上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话,尝试删除 `objectKey` 对应的临时 blob,然后标记为 `EXPIRED`。已完成文件不参与清理,避免误删已经落库的生产对象。
|
- 2026-04-08 阶段 3 第四小步补充:上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话,尝试删除 `objectKey` 对应的临时 blob,然后标记为 `EXPIRED`。已完成文件不参与清理,避免误删已经落库的生产对象。
|
||||||
- 2026-04-08 阶段 4 第一小步补充:后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略,并把当前 `FileStorageProperties` 映射为 `LOCAL` 或 `S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON;当前能力声明中 `multipartUpload=false`,用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID,但 `FileContentStorage` 仍保持单对象上传/校验抽象,旧 `/api/files/**` 生产路径不切换。
|
- 2026-04-08 阶段 4 第一小步补充:后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略,并把当前 `FileStorageProperties` 映射为 `LOCAL` 或 `S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON;当前能力声明中 `multipartUpload=false`,用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID,但 `FileContentStorage` 仍保持单对象上传/校验抽象,旧 `/api/files/**` 生产路径不切换。
|
||||||
- 2026-04-08 `files/storage` 合并补充:S3 存储实现拆出多吉云临时密钥客户端与运行期会话提供器。`S3FileContentStorage` 现在通过 `S3SessionProvider.currentSession()` 获取当前 bucket、`S3Client` 和 `S3Presigner`,避免每次操作重复内联多吉云 token 解析逻辑;测试环境可直接注入 mock S3 client/presigner。该改动没有引入 multipart,仍是单对象 PUT/HEAD/GET/COPY/DELETE 路径。
|
- 2026-04-08 `files/storage` 合并补充:S3 存储实现拆出多吉云临时密钥客户端与运行期会话提供器。`S3FileContentStorage` 现在通过 `S3SessionProvider.currentSession()` 获取当前 bucket、`S3Client` 和 `S3Presigner`,避免每次操作重复内联多吉云 token 解析逻辑;测试环境可直接注入 mock S3 client/presigner。该改动没有引入 multipart,仍是单对象 PUT/HEAD/GET/COPY/DELETE 路径。
|
||||||
|
- 2026-04-08 阶段 4 第二小步补充:`FileService` 在创建新的 `FileEntity.VERSION` 时会通过 `StoragePolicyService.ensureDefaultPolicy()` 写入默认 `storagePolicyId`;`FileEntityBackfillService` 对历史 `FileBlob` 回填新实体时也写入同一默认策略。复用已有实体时保持原策略字段不变,只增加引用计数,避免在兼容迁移阶段覆盖历史数据。
|
||||||
|
|||||||
@@ -166,3 +166,4 @@
|
|||||||
- 2026-04-08 multipart 评估结论:暂不把 v2 上传会话直接接入真实对象存储分片写入/合并。当前 `FileContentStorage` 仍是单对象上传/校验抽象,缺少 multipart uploadId、part URL 预签名、complete/abort 语义;立即接入会把上传会话写死在当前多吉云 S3 配置上,并让过期清理误以为 `deleteBlob` 能释放未完成分片。下一步先做阶段 4 存储策略与能力声明骨架,再按 `multipartUpload` 能力接 S3 multipart。
|
- 2026-04-08 multipart 评估结论:暂不把 v2 上传会话直接接入真实对象存储分片写入/合并。当前 `FileContentStorage` 仍是单对象上传/校验抽象,缺少 multipart uploadId、part URL 预签名、complete/abort 语义;立即接入会把上传会话写死在当前多吉云 S3 配置上,并让过期清理误以为 `deleteBlob` 能释放未完成分片。下一步先做阶段 4 存储策略与能力声明骨架,再按 `multipartUpload` 能力接 S3 multipart。
|
||||||
- 2026-04-08 阶段 4 第一小步:新增 `StoragePolicy`、`StoragePolicyType`、`StoragePolicyCredentialMode`、`StoragePolicyCapabilities` 与 `StoragePolicyService`,启动时把当前 `app.storage.provider` 映射成一条默认策略;本地策略声明 `serverProxyDownload=true`、`multipartUpload=false`,多吉云/S3 兼容策略声明 `directUpload=true`、`signedDownloadUrl=true`、`requiresCors=true`、`multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`,但旧上传下载路径和前端上传队列仍未切换。
|
- 2026-04-08 阶段 4 第一小步:新增 `StoragePolicy`、`StoragePolicyType`、`StoragePolicyCredentialMode`、`StoragePolicyCapabilities` 与 `StoragePolicyService`,启动时把当前 `app.storage.provider` 映射成一条默认策略;本地策略声明 `serverProxyDownload=true`、`multipartUpload=false`,多吉云/S3 兼容策略声明 `directUpload=true`、`signedDownloadUrl=true`、`requiresCors=true`、`multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`,但旧上传下载路径和前端上传队列仍未切换。
|
||||||
- 2026-04-08 合并 `files/storage` 补提交后修复:`S3FileContentStorage` 改为复用 `DogeCloudS3SessionProvider` / `DogeCloudTmpTokenClient` 获取并缓存运行期 `S3Client` 与 `S3Presigner`,保留生产构造器 `S3FileContentStorage(FileStorageProperties)`,同时提供测试用注入构造器;S3 直传、签名下载、上传校验、读旧对象键 fallback、rename/move/copy、离线快传对象读写继续通过 `FileContentStorage` 统一抽象。
|
- 2026-04-08 合并 `files/storage` 补提交后修复:`S3FileContentStorage` 改为复用 `DogeCloudS3SessionProvider` / `DogeCloudTmpTokenClient` 获取并缓存运行期 `S3Client` 与 `S3Presigner`,保留生产构造器 `S3FileContentStorage(FileStorageProperties)`,同时提供测试用注入构造器;S3 直传、签名下载、上传校验、读旧对象键 fallback、rename/move/copy、离线快传对象读写继续通过 `FileContentStorage` 统一抽象。
|
||||||
|
- 2026-04-08 阶段 4 第二小步:新写入和回填生成的 `FileEntity.VERSION` 会记录默认 `StoragePolicy.id` 到 `storagePolicyId`,让物理实体可以追踪归属存储策略;复用已有 `FileEntity` 时只增加引用计数,不覆盖历史实体策略字段。旧 `/api/files/**` 读取路径仍继续依赖 `StoredFile.blob`。
|
||||||
|
|||||||
Reference in New Issue
Block a user