Compare commits

..

2 Commits

Author SHA1 Message Date
yoyuzh
9af2d38e37 Eliminate admin N+1 queries and lazy-load app routes 2026-04-12 00:48:23 +08:00
yoyuzh
30a9bbc1e7 Refactor backend and frontend modules for architecture alignment 2026-04-12 00:32:21 +08:00
256 changed files with 25708 additions and 4807 deletions

5
.claude/settings.json Normal file
View File

@@ -0,0 +1,5 @@
{
"enabledPlugins": {
"codex@openai-codex": true
}
}

View File

@@ -0,0 +1,38 @@
{
"name": "my_site backend",
"image": "mcr.microsoft.com/devcontainers/java:1-17-bullseye",
"remoteUser": "vscode",
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}/backend",
"runArgs": [
"--init"
],
"mounts": [
"source=my-site-backend-maven-cache,target=/home/vscode/.m2,type=volume"
],
"containerEnv": {
"APP_JWT_SECRET": "devcontainer-local-jwt-secret-please-change-if-needed",
"SPRING_PROFILES_ACTIVE": "dev"
},
"forwardPorts": [
8080
],
"portsAttributes": {
"8080": {
"label": "Spring Boot backend",
"onAutoForward": "notify"
}
},
"customizations": {
"vscode": {
"extensions": [
"vscjava.vscode-java-pack",
"vscjava.vscode-lombok",
"vmware.vscode-spring-boot"
],
"settings": {
"java.import.maven.enabled": true,
"java.configuration.updateBuildConfiguration": "interactive"
}
}
}
}

View File

@@ -6,7 +6,8 @@ This repository is split across a Java backend, a Vite/React frontend, a small `
- Every new window / new session that starts work in this repository must read `memory.md`, `docs/architecture.md`, and `docs/api-reference.md` first before planning, coding, reviewing, or deploying. - Every new window / new session that starts work in this repository must read `memory.md`, `docs/architecture.md`, and `docs/api-reference.md` first before planning, coding, reviewing, or deploying.
- Treat `memory.md` as the current project memory and continuity handoff unless the user explicitly overrides it. - Treat `memory.md` as the current project memory and continuity handoff unless the user explicitly overrides it.
- Treat `docs/architecture.md` as the system-level source of truth for module boundaries and runtime structure. - Treat `docs/architecture.md` as the project architecture document and source of truth for module boundaries and runtime structure.
- Do not edit `docs/architecture.md` during normal implementation, refactor, review, or handoff work. Only change it when the user explicitly asks to update the architecture document itself.
- Treat `docs/api-reference.md` as the quick reference for backend endpoints and auth/public access boundaries. - Treat `docs/api-reference.md` as the quick reference for backend endpoints and auth/public access boundaries.
## Real project structure ## Real project structure
@@ -117,7 +118,8 @@ Important:
### Project memory upkeep ### Project memory upkeep
- Every time a task causes a major project change, update `memory.md` and `docs/architecture.md` in the same turn before handing off. Major changes include architecture shifts, storage/provider migrations, auth or security model changes, deployment topology changes, and meaningful new product capabilities. - Every time a task causes a major project change, update `memory.md` in the same turn before handing off.
- Do not update `docs/architecture.md` as part of routine implementation follow-up. That file is reserved for explicit architecture-document changes requested by the user.
## Repo-specific guardrails ## Repo-specific guardrails
@@ -129,4 +131,11 @@ Important:
- For frontend releases, prefer `node scripts/deploy-front-oss.mjs` over ad hoc `ossutil` or manual uploads. - For frontend releases, prefer `node scripts/deploy-front-oss.mjs` over ad hoc `ossutil` or manual uploads.
- For backend releases, package from `backend/` and deploy the produced jar; do not commit `backend/target/` artifacts to git unless the user explicitly asks for that unusual workflow. - For backend releases, package from `backend/` and deploy the produced jar; do not commit `backend/target/` artifacts to git unless the user explicitly asks for that unusual workflow.
## Debugging Discipline
- When diagnosing environment or download issues, use short probes first: prefer `curl --max-time`, `mvn -q`, `apt-get update`, `mvn dependency:get`, or similar bounded checks before any full build or long download.
- Do not wait indefinitely on a stalled download or network command. If a command shows no progress within a short probe window, stop and inspect the active proxy, DNS, and mirror path before retrying.
- For WSL-based debugging, prefer the native WSL shell plus the current mirror/proxy settings already in place. If a download path is slow, verify whether the proxy path is actually faster before forcing direct access.
- If a package source is unstable, switch to a domestic mirror only after confirming whether the failure is in DNS, proxy routing, or the upstream mirror itself.
Directory-level `AGENTS.md` files in `backend/`, `front/`, and `docs/` add more specific rules and override this file where they are more specific. Directory-level `AGENTS.md` files in `backend/`, `front/`, and `docs/` add more specific rules and override this file where they are more specific.

View File

@@ -13,6 +13,26 @@
- Maven 3.9+ - Maven 3.9+
- 生产环境使用 MySQL 8.x 或 openGauss - 生产环境使用 MySQL 8.x 或 openGauss
## Dev Container
仓库根目录现在提供了后端专用的 `.devcontainer/devcontainer.json`
用途:
- 使用 JDK 17 + Maven 的开发容器打开后端
- 默认把工作目录定位到 `backend/`
- 默认转发 `8080`
- 默认注入一个本地开发用 `APP_JWT_SECRET`
- 默认启用 `SPRING_PROFILES_ACTIVE=dev`
进入容器后,仍按仓库已有命令运行:
```bash
mvn spring-boot:run -Dspring-boot.run.profiles=dev
```
如果你需要读取仓库根目录的 `.env`,它仍然和 `backend/` 一起挂载在同一个 workspace 下,可通过 `../.env` 访问。
## 启动 ## 启动
推荐先在仓库根目录准备并加载 `.env` 推荐先在仓库根目录准备并加载 `.env`

View File

@@ -0,0 +1,190 @@
2026-04-09 00:35:10.239730+08:00 jdbc[4]: exception
org.h2.jdbc.JdbcSQLSyntaxErrorException: Column "SF1_0.IS_RECYCLE_ROOT" not found; SQL statement:
select sf1_0.id,b1_0.id,b1_0.content_type,b1_0.created_at,b1_0.object_key,b1_0.size,sf1_0.content_type,sf1_0.created_at,sf1_0.deleted_at,sf1_0.is_directory,sf1_0.filename,sf1_0.storage_name,sf1_0.path,sf1_0.primary_entity_id,sf1_0.recycle_group_id,sf1_0.recycle_original_path,sf1_0.is_recycle_root,sf1_0.size,sf1_0.updated_at,sf1_0.user_id from portal_file sf1_0 left join portal_file_blob b1_0 on b1_0.id=sf1_0.blob_id where sf1_0.deleted_at is not null and sf1_0.deleted_at<? order by sf1_0.deleted_at [42122-224]
at org.h2.message.DbException.getJdbcSQLException(DbException.java:514)
at org.h2.message.DbException.getJdbcSQLException(DbException.java:489)
at org.h2.message.DbException.get(DbException.java:223)
at org.h2.message.DbException.get(DbException.java:199)
at org.h2.expression.ExpressionColumn.getColumnException(ExpressionColumn.java:244)
at org.h2.expression.ExpressionColumn.optimizeOther(ExpressionColumn.java:226)
at org.h2.expression.ExpressionColumn.optimize(ExpressionColumn.java:213)
at org.h2.command.query.Select.optimizeExpressionsAndPreserveAliases(Select.java:1285)
at org.h2.command.query.Select.prepareExpressions(Select.java:1167)
at org.h2.command.query.Query.prepare(Query.java:218)
at org.h2.command.Parser.prepareCommand(Parser.java:489)
at org.h2.engine.SessionLocal.prepareLocal(SessionLocal.java:639)
at org.h2.engine.SessionLocal.prepareCommand(SessionLocal.java:559)
at org.h2.jdbc.JdbcConnection.prepareCommand(JdbcConnection.java:1166)
at org.h2.jdbc.JdbcPreparedStatement.<init>(JdbcPreparedStatement.java:93)
at org.h2.jdbc.JdbcConnection.prepareStatement(JdbcConnection.java:316)
at com.zaxxer.hikari.pool.ProxyConnection.prepareStatement(ProxyConnection.java:328)
at com.zaxxer.hikari.pool.HikariProxyConnection.prepareStatement(HikariProxyConnection.java)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$5.doPrepare(StatementPreparerImpl.java:153)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$StatementPreparationTemplate.prepareStatement(StatementPreparerImpl.java:183)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl.prepareQueryStatement(StatementPreparerImpl.java:155)
at org.hibernate.sql.exec.spi.JdbcSelectExecutor.lambda$list$0(JdbcSelectExecutor.java:85)
at org.hibernate.sql.results.jdbc.internal.DeferredResultSetAccess.executeQuery(DeferredResultSetAccess.java:231)
at org.hibernate.sql.results.jdbc.internal.DeferredResultSetAccess.getResultSet(DeferredResultSetAccess.java:167)
at org.hibernate.sql.results.jdbc.internal.JdbcValuesResultSetImpl.advanceNext(JdbcValuesResultSetImpl.java:265)
at org.hibernate.sql.results.jdbc.internal.JdbcValuesResultSetImpl.processNext(JdbcValuesResultSetImpl.java:145)
at org.hibernate.sql.results.jdbc.internal.AbstractJdbcValues.next(AbstractJdbcValues.java:19)
at org.hibernate.sql.results.internal.RowProcessingStateStandardImpl.next(RowProcessingStateStandardImpl.java:67)
at org.hibernate.sql.results.spi.ListResultsConsumer.consume(ListResultsConsumer.java:204)
at org.hibernate.sql.results.spi.ListResultsConsumer.consume(ListResultsConsumer.java:33)
at org.hibernate.sql.exec.internal.JdbcSelectExecutorStandardImpl.doExecuteQuery(JdbcSelectExecutorStandardImpl.java:211)
at org.hibernate.sql.exec.internal.JdbcSelectExecutorStandardImpl.executeQuery(JdbcSelectExecutorStandardImpl.java:83)
at org.hibernate.sql.exec.spi.JdbcSelectExecutor.list(JdbcSelectExecutor.java:76)
at org.hibernate.sql.exec.spi.JdbcSelectExecutor.list(JdbcSelectExecutor.java:65)
at org.hibernate.query.sqm.internal.ConcreteSqmSelectQueryPlan.lambda$new$2(ConcreteSqmSelectQueryPlan.java:139)
at org.hibernate.query.sqm.internal.ConcreteSqmSelectQueryPlan.withCacheableSqmInterpretation(ConcreteSqmSelectQueryPlan.java:382)
at org.hibernate.query.sqm.internal.ConcreteSqmSelectQueryPlan.performList(ConcreteSqmSelectQueryPlan.java:302)
at org.hibernate.query.sqm.internal.QuerySqmImpl.doList(QuerySqmImpl.java:526)
at org.hibernate.query.spi.AbstractSelectionQuery.list(AbstractSelectionQuery.java:423)
at org.hibernate.query.Query.getResultList(Query.java:120)
at org.springframework.data.jpa.repository.query.JpaQueryExecution$CollectionExecution.doExecute(JpaQueryExecution.java:130)
at org.springframework.data.jpa.repository.query.JpaQueryExecution.execute(JpaQueryExecution.java:93)
at org.springframework.data.jpa.repository.query.AbstractJpaQuery.doExecute(AbstractJpaQuery.java:152)
at org.springframework.data.jpa.repository.query.AbstractJpaQuery.execute(AbstractJpaQuery.java:140)
at org.springframework.data.repository.core.support.RepositoryMethodInvoker.doInvoke(RepositoryMethodInvoker.java:170)
at org.springframework.data.repository.core.support.RepositoryMethodInvoker.invoke(RepositoryMethodInvoker.java:158)
at org.springframework.data.repository.core.support.QueryExecutorMethodInterceptor.doInvoke(QueryExecutorMethodInterceptor.java:169)
at org.springframework.data.repository.core.support.QueryExecutorMethodInterceptor.invoke(QueryExecutorMethodInterceptor.java:148)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.data.projection.DefaultMethodInvokingMethodInterceptor.invoke(DefaultMethodInvokingMethodInterceptor.java:70)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:379)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:138)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodInterceptor.invoke(CrudMethodMetadataPostProcessor.java:136)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:223)
at jdk.proxy2/jdk.proxy2.$Proxy162.findByDeletedAtBefore(Unknown Source)
at com.yoyuzh.files.FileService.pruneExpiredRecycleBinItems(FileService.java:292)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:355)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:196)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:768)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:379)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:768)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:720)
at com.yoyuzh.files.FileService$$SpringCGLIB$$0.pruneExpiredRecycleBinItems(<generated>)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.springframework.scheduling.support.ScheduledMethodRunnable.runInternal(ScheduledMethodRunnable.java:130)
at org.springframework.scheduling.support.ScheduledMethodRunnable.lambda$run$2(ScheduledMethodRunnable.java:124)
at io.micrometer.observation.Observation.observe(Observation.java:499)
at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:124)
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
2026-04-09 00:35:10.299570+08:00 jdbc[3]: exception
org.h2.jdbc.JdbcSQLSyntaxErrorException: Column "SF1_0.IS_RECYCLE_ROOT" not found; SQL statement:
select sf1_0.id,sf1_0.blob_id,sf1_0.content_type,sf1_0.created_at,sf1_0.deleted_at,sf1_0.is_directory,sf1_0.filename,sf1_0.storage_name,sf1_0.path,sf1_0.primary_entity_id,sf1_0.recycle_group_id,sf1_0.recycle_original_path,sf1_0.is_recycle_root,sf1_0.size,sf1_0.updated_at,sf1_0.user_id from portal_file sf1_0 where not(sf1_0.is_directory) and sf1_0.blob_id is null [42122-224]
at org.h2.message.DbException.getJdbcSQLException(DbException.java:514)
at org.h2.message.DbException.getJdbcSQLException(DbException.java:489)
at org.h2.message.DbException.get(DbException.java:223)
at org.h2.message.DbException.get(DbException.java:199)
at org.h2.expression.ExpressionColumn.getColumnException(ExpressionColumn.java:244)
at org.h2.expression.ExpressionColumn.optimizeOther(ExpressionColumn.java:226)
at org.h2.expression.ExpressionColumn.optimize(ExpressionColumn.java:213)
at org.h2.command.query.Select.optimizeExpressionsAndPreserveAliases(Select.java:1285)
at org.h2.command.query.Select.prepareExpressions(Select.java:1167)
at org.h2.command.query.Query.prepare(Query.java:218)
at org.h2.command.Parser.prepareCommand(Parser.java:489)
at org.h2.engine.SessionLocal.prepareLocal(SessionLocal.java:639)
at org.h2.engine.SessionLocal.prepareCommand(SessionLocal.java:559)
at org.h2.jdbc.JdbcConnection.prepareCommand(JdbcConnection.java:1166)
at org.h2.jdbc.JdbcPreparedStatement.<init>(JdbcPreparedStatement.java:93)
at org.h2.jdbc.JdbcConnection.prepareStatement(JdbcConnection.java:316)
at com.zaxxer.hikari.pool.ProxyConnection.prepareStatement(ProxyConnection.java:328)
at com.zaxxer.hikari.pool.HikariProxyConnection.prepareStatement(HikariProxyConnection.java)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$5.doPrepare(StatementPreparerImpl.java:153)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$StatementPreparationTemplate.prepareStatement(StatementPreparerImpl.java:183)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl.prepareQueryStatement(StatementPreparerImpl.java:155)
at org.hibernate.sql.exec.spi.JdbcSelectExecutor.lambda$list$0(JdbcSelectExecutor.java:85)
at org.hibernate.sql.results.jdbc.internal.DeferredResultSetAccess.executeQuery(DeferredResultSetAccess.java:231)
at org.hibernate.sql.results.jdbc.internal.DeferredResultSetAccess.getResultSet(DeferredResultSetAccess.java:167)
at org.hibernate.sql.results.jdbc.internal.JdbcValuesResultSetImpl.advanceNext(JdbcValuesResultSetImpl.java:265)
at org.hibernate.sql.results.jdbc.internal.JdbcValuesResultSetImpl.processNext(JdbcValuesResultSetImpl.java:145)
at org.hibernate.sql.results.jdbc.internal.AbstractJdbcValues.next(AbstractJdbcValues.java:19)
at org.hibernate.sql.results.internal.RowProcessingStateStandardImpl.next(RowProcessingStateStandardImpl.java:67)
at org.hibernate.sql.results.spi.ListResultsConsumer.consume(ListResultsConsumer.java:204)
at org.hibernate.sql.results.spi.ListResultsConsumer.consume(ListResultsConsumer.java:33)
at org.hibernate.sql.exec.internal.JdbcSelectExecutorStandardImpl.doExecuteQuery(JdbcSelectExecutorStandardImpl.java:211)
at org.hibernate.sql.exec.internal.JdbcSelectExecutorStandardImpl.executeQuery(JdbcSelectExecutorStandardImpl.java:83)
at org.hibernate.sql.exec.spi.JdbcSelectExecutor.list(JdbcSelectExecutor.java:76)
at org.hibernate.sql.exec.spi.JdbcSelectExecutor.list(JdbcSelectExecutor.java:65)
at org.hibernate.query.sqm.internal.ConcreteSqmSelectQueryPlan.lambda$new$2(ConcreteSqmSelectQueryPlan.java:139)
at org.hibernate.query.sqm.internal.ConcreteSqmSelectQueryPlan.withCacheableSqmInterpretation(ConcreteSqmSelectQueryPlan.java:382)
at org.hibernate.query.sqm.internal.ConcreteSqmSelectQueryPlan.performList(ConcreteSqmSelectQueryPlan.java:302)
at org.hibernate.query.sqm.internal.QuerySqmImpl.doList(QuerySqmImpl.java:526)
at org.hibernate.query.spi.AbstractSelectionQuery.list(AbstractSelectionQuery.java:423)
at org.hibernate.query.Query.getResultList(Query.java:120)
at org.springframework.data.jpa.repository.query.JpaQueryExecution$CollectionExecution.doExecute(JpaQueryExecution.java:130)
at org.springframework.data.jpa.repository.query.JpaQueryExecution.execute(JpaQueryExecution.java:93)
at org.springframework.data.jpa.repository.query.AbstractJpaQuery.doExecute(AbstractJpaQuery.java:152)
at org.springframework.data.jpa.repository.query.AbstractJpaQuery.execute(AbstractJpaQuery.java:140)
at org.springframework.data.repository.core.support.RepositoryMethodInvoker.doInvoke(RepositoryMethodInvoker.java:170)
at org.springframework.data.repository.core.support.RepositoryMethodInvoker.invoke(RepositoryMethodInvoker.java:158)
at org.springframework.data.repository.core.support.QueryExecutorMethodInterceptor.doInvoke(QueryExecutorMethodInterceptor.java:169)
at org.springframework.data.repository.core.support.QueryExecutorMethodInterceptor.invoke(QueryExecutorMethodInterceptor.java:148)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.data.projection.DefaultMethodInvokingMethodInterceptor.invoke(DefaultMethodInvokingMethodInterceptor.java:70)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:379)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:138)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodInterceptor.invoke(CrudMethodMetadataPostProcessor.java:136)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:223)
at jdk.proxy2/jdk.proxy2.$Proxy162.findAllByDirectoryFalseAndBlobIsNull(Unknown Source)
at com.yoyuzh.files.FileBlobBackfillService.backfillMissingBlobs(FileBlobBackfillService.java:28)
at com.yoyuzh.files.FileBlobBackfillService.run(FileBlobBackfillService.java:23)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:355)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:196)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:768)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:379)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:184)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:768)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:720)
at com.yoyuzh.files.FileBlobBackfillService$$SpringCGLIB$$0.run(<generated>)
at org.springframework.boot.SpringApplication.lambda$callRunner$5(SpringApplication.java:790)
at org.springframework.util.function.ThrowingConsumer$1.acceptWithException(ThrowingConsumer.java:82)
at org.springframework.util.function.ThrowingConsumer.accept(ThrowingConsumer.java:60)
at org.springframework.util.function.ThrowingConsumer$1.accept(ThrowingConsumer.java:86)
at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:798)
at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:789)
at org.springframework.boot.SpringApplication.lambda$callRunners$3(SpringApplication.java:774)
at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:184)
at java.base/java.util.stream.SortedOps$SizedRefSortingSink.end(SortedOps.java:357)
at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:557)
at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:546)
at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151)
at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174)
at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:611)
at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:774)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:342)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1363)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1352)
at com.yoyuzh.PortalBackendApplication.main(PortalBackendApplication.java:25)

View File

@@ -38,6 +38,14 @@
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId> <artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.springdoc</groupId> <groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId> <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>

View File

@@ -1,6 +1,7 @@
package com.yoyuzh; package com.yoyuzh;
import com.yoyuzh.config.AdminProperties; import com.yoyuzh.config.AdminProperties;
import com.yoyuzh.config.AppRedisProperties;
import com.yoyuzh.config.AndroidReleaseProperties; import com.yoyuzh.config.AndroidReleaseProperties;
import com.yoyuzh.config.CorsProperties; import com.yoyuzh.config.CorsProperties;
import com.yoyuzh.config.FileStorageProperties; import com.yoyuzh.config.FileStorageProperties;
@@ -17,7 +18,8 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
CorsProperties.class, CorsProperties.class,
AdminProperties.class, AdminProperties.class,
RegistrationProperties.class, RegistrationProperties.class,
AndroidReleaseProperties.class AndroidReleaseProperties.class,
AppRedisProperties.class
}) })
public class PortalBackendApplication { public class PortalBackendApplication {

View File

@@ -1,27 +1,34 @@
package com.yoyuzh.admin; package com.yoyuzh.admin;
import com.yoyuzh.config.AdminProperties; import com.yoyuzh.auth.UserRole;
import org.springframework.security.core.Authentication; import org.springframework.security.core.Authentication;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.Set; import java.util.Objects;
import java.util.stream.Collectors;
@Component @Component
public class AdminAccessEvaluator { public class AdminAccessEvaluator {
private final Set<String> adminUsernames;
public AdminAccessEvaluator(AdminProperties adminProperties) {
this.adminUsernames = adminProperties.getUsernames().stream()
.map(username -> username == null ? "" : username.trim())
.filter(username -> !username.isEmpty())
.collect(Collectors.toUnmodifiableSet());
}
public boolean isAdmin(Authentication authentication) { public boolean isAdmin(Authentication authentication) {
return authentication != null if (authentication == null || !authentication.isAuthenticated()) {
&& authentication.isAuthenticated() return false;
&& adminUsernames.contains(authentication.getName()); }
return authentication.getAuthorities().stream()
.map(GrantedAuthority::getAuthority)
.map(this::toUserRole)
.filter(Objects::nonNull)
.anyMatch(UserRole::canAccessAdmin);
}
private UserRole toUserRole(String authority) {
if (authority == null || !authority.startsWith("ROLE_")) {
return null;
}
try {
return UserRole.valueOf(authority.substring("ROLE_".length()));
} catch (IllegalArgumentException ex) {
return null;
}
} }
} }

View File

@@ -0,0 +1,19 @@
package com.yoyuzh.admin;
public enum AdminAuditAction {
UPDATE_REGISTRATION_INVITE_CODE,
ROTATE_REGISTRATION_INVITE_CODE,
UPDATE_OFFLINE_TRANSFER_STORAGE_LIMIT,
UPDATE_USER_ROLE,
UPDATE_USER_BANNED,
UPDATE_USER_PASSWORD,
RESET_USER_PASSWORD,
UPDATE_USER_STORAGE_QUOTA,
UPDATE_USER_MAX_UPLOAD_SIZE,
DELETE_SHARE,
DELETE_FILE,
CREATE_STORAGE_POLICY,
UPDATE_STORAGE_POLICY,
UPDATE_STORAGE_POLICY_STATUS,
CREATE_STORAGE_POLICY_MIGRATION_TASK
}

View File

@@ -0,0 +1,126 @@
package com.yoyuzh.admin;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.PrePersist;
import jakarta.persistence.Table;
import java.time.LocalDateTime;
@Entity
@Table(name = "portal_admin_audit_log")
public class AdminAuditLogEntity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column(name = "actor_user_id")
private Long actorUserId;
@Column(name = "actor_username", nullable = false, length = 100)
private String actorUsername;
@Column(name = "actor_authorities", nullable = false, length = 255)
private String actorAuthorities;
@Column(name = "action_type", nullable = false, length = 100)
private String actionType;
@Column(name = "target_type", nullable = false, length = 100)
private String targetType;
@Column(name = "target_id")
private Long targetId;
@Column(name = "summary", nullable = false, length = 255)
private String summary;
@Column(name = "details_json", nullable = false, columnDefinition = "TEXT")
private String detailsJson;
@Column(name = "created_at", nullable = false)
private LocalDateTime createdAt;
@PrePersist
public void prePersist() {
if (createdAt == null) {
createdAt = LocalDateTime.now();
}
}
public Long getId() {
return id;
}
public Long getActorUserId() {
return actorUserId;
}
public void setActorUserId(Long actorUserId) {
this.actorUserId = actorUserId;
}
public String getActorUsername() {
return actorUsername;
}
public void setActorUsername(String actorUsername) {
this.actorUsername = actorUsername;
}
public String getActorAuthorities() {
return actorAuthorities;
}
public void setActorAuthorities(String actorAuthorities) {
this.actorAuthorities = actorAuthorities;
}
public String getActionType() {
return actionType;
}
public void setActionType(String actionType) {
this.actionType = actionType;
}
public String getTargetType() {
return targetType;
}
public void setTargetType(String targetType) {
this.targetType = targetType;
}
public Long getTargetId() {
return targetId;
}
public void setTargetId(Long targetId) {
this.targetId = targetId;
}
public String getSummary() {
return summary;
}
public void setSummary(String summary) {
this.summary = summary;
}
public String getDetailsJson() {
return detailsJson;
}
public void setDetailsJson(String detailsJson) {
this.detailsJson = detailsJson;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
}

View File

@@ -0,0 +1,25 @@
package com.yoyuzh.admin;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
public interface AdminAuditLogRepository extends JpaRepository<AdminAuditLogEntity, Long> {
@Query("""
select l from AdminAuditLogEntity l
where (:actorQuery = '' or lower(l.actorUsername) like lower(concat('%', :actorQuery, '%')))
and (:actionType = '' or l.actionType = :actionType)
and (:targetType = '' or l.targetType = :targetType)
and (:targetId is null or l.targetId = :targetId)
""")
Page<AdminAuditLogEntity> search(
@Param("actorQuery") String actorQuery,
@Param("actionType") String actionType,
@Param("targetType") String targetType,
@Param("targetId") Long targetId,
Pageable pageable
);
}

View File

@@ -0,0 +1,17 @@
package com.yoyuzh.admin;
import java.time.LocalDateTime;
public record AdminAuditLogResponse(
Long id,
Long actorUserId,
String actorUsername,
String actorAuthorities,
String actionType,
String targetType,
Long targetId,
String summary,
String detailsJson,
LocalDateTime createdAt
) {
}

View File

@@ -0,0 +1,59 @@
package com.yoyuzh.admin;
import com.yoyuzh.common.PageResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class AdminAuditQueryService {
private final AdminAuditLogRepository adminAuditLogRepository;
public PageResponse<AdminAuditLogResponse> listAuditLogs(int page,
int size,
String actorQuery,
String actionType,
String targetType,
Long targetId) {
Page<AdminAuditLogEntity> result = adminAuditLogRepository.search(
normalizeQuery(actorQuery),
normalizeQuery(actionType),
normalizeQuery(targetType),
targetId,
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt")
.and(Sort.by(Sort.Direction.DESC, "id")))
);
return new PageResponse<>(
result.getContent().stream().map(this::toResponse).toList(),
result.getTotalElements(),
page,
size
);
}
private AdminAuditLogResponse toResponse(AdminAuditLogEntity entity) {
return new AdminAuditLogResponse(
entity.getId(),
entity.getActorUserId(),
entity.getActorUsername(),
entity.getActorAuthorities(),
entity.getActionType(),
entity.getTargetType(),
entity.getTargetId(),
entity.getSummary(),
entity.getDetailsJson(),
entity.getCreatedAt()
);
}
private String normalizeQuery(String query) {
if (query == null) {
return "";
}
return query.trim();
}
}

View File

@@ -0,0 +1,73 @@
package com.yoyuzh.admin;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.auth.UserRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class AdminAuditService {
private final AdminAuditLogRepository adminAuditLogRepository;
private final UserRepository userRepository;
private final ObjectMapper objectMapper;
public void record(AdminAuditAction action,
String targetType,
Long targetId,
String summary,
Map<String, Object> details) {
ActorSnapshot actor = resolveActorSnapshot();
AdminAuditLogEntity entity = new AdminAuditLogEntity();
entity.setActorUserId(actor.userId());
entity.setActorUsername(actor.username());
entity.setActorAuthorities(actor.authorities());
entity.setActionType(action.name());
entity.setTargetType(targetType);
entity.setTargetId(targetId);
entity.setSummary(summary);
entity.setDetailsJson(serializeDetails(details));
adminAuditLogRepository.save(entity);
}
private ActorSnapshot resolveActorSnapshot() {
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
if (authentication == null || !authentication.isAuthenticated()) {
return new ActorSnapshot(null, "system", "");
}
String username = authentication.getName();
Long userId = StringUtils.hasText(username)
? userRepository.findByUsername(username).map(user -> user.getId()).orElse(null)
: null;
String authorities = authentication.getAuthorities() == null
? ""
: authentication.getAuthorities().stream()
.map(GrantedAuthority::getAuthority)
.sorted()
.collect(Collectors.joining(","));
if (!StringUtils.hasText(username)) {
return new ActorSnapshot(userId, "system", authorities);
}
return new ActorSnapshot(userId, username, authorities);
}
private String serializeDetails(Map<String, Object> details) {
try {
return objectMapper.writeValueAsString(details == null ? Map.of() : details);
} catch (JsonProcessingException ex) {
return "{}";
}
}
private record ActorSnapshot(Long userId, String username, String authorities) {
}
}

View File

@@ -0,0 +1,121 @@
package com.yoyuzh.admin;
import com.yoyuzh.auth.RegistrationInviteService;
import com.yoyuzh.auth.UserRole;
import com.yoyuzh.config.AppRedisProperties;
import com.yoyuzh.config.FileStorageProperties;
import com.yoyuzh.config.JwtProperties;
import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.core.FileEntityRepository;
import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyCapabilities;
import com.yoyuzh.files.policy.StoragePolicyService;
import lombok.RequiredArgsConstructor;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.util.List;
@Service
@RequiredArgsConstructor
public class AdminConfigSnapshotService {
private final RegistrationInviteService registrationInviteService;
private final AdminMetricsService adminMetricsService;
private final AppRedisProperties redisProperties;
private final FileStorageProperties fileStorageProperties;
private final JwtProperties jwtProperties;
private final Environment environment;
private final StoragePolicyService storagePolicyService;
private final StoredFileRepository storedFileRepository;
private final FileBlobRepository fileBlobRepository;
private final FileEntityRepository fileEntityRepository;
public AdminSettingsResponse getSettings() {
return new AdminSettingsResponse(
new AdminSettingsResponse.SiteSection(false, false),
new AdminSettingsResponse.RegistrationSection(
true,
registrationInviteService.getCurrentInviteCode(),
List.of(UserRole.MODERATOR.name(), UserRole.ADMIN.name()),
true
),
new AdminSettingsResponse.UserSessionSection(
jwtProperties.getAccessExpirationSeconds(),
jwtProperties.getRefreshExpirationSeconds(),
redisProperties.isEnabled(),
redisProperties.getTtlBufferSeconds(),
false
),
new AdminSettingsResponse.TransferSection(
adminMetricsService.getOfflineTransferStorageLimitBytes(),
true
),
new AdminSettingsResponse.MediaProcessingSection(true, false, false, false),
new AdminSettingsResponse.QueueSection(
redisProperties.isEnabled() ? "redis" : "in-memory",
readLongProperty("app.redis.broker.media-meta.fixed-delay-ms", 3000L),
readLongProperty("app.redis.broker.media-meta.initial-delay-ms", 15000L),
false
),
new AdminSettingsResponse.AppearanceSection(false, false),
new AdminSettingsResponse.ServerSection(
normalizeStorageProvider(fileStorageProperties.getProvider()),
redisProperties.isEnabled(),
false
)
);
}
public AdminFilesystemResponse getFilesystem() {
StoragePolicy defaultPolicy = storagePolicyService.ensureDefaultPolicy();
StoragePolicyCapabilities capabilities = storagePolicyService.readCapabilities(defaultPolicy);
boolean directUpload = capabilities.directUpload();
return new AdminFilesystemResponse(
new AdminFilesystemResponse.OverviewSection(
normalizeStorageProvider(fileStorageProperties.getProvider()),
storedFileRepository.count(),
fileBlobRepository.count(),
fileEntityRepository.count()
),
AdminStoragePolicyResponses.from(storagePolicyService, defaultPolicy),
new AdminFilesystemResponse.UploadSection(
!directUpload,
directUpload && !capabilities.multipartUpload(),
directUpload && capabilities.multipartUpload(),
resolveEffectiveMaxFileSize(defaultPolicy, capabilities)
),
new AdminFilesystemResponse.MediaProcessingSection(true, capabilities.thumbnailNative()),
new AdminFilesystemResponse.CacheSection(
redisProperties.isEnabled() ? "redis" : "disabled",
redisProperties.getCache().getFilesListTtlSeconds(),
redisProperties.getCache().getDirectoryVersionTtlSeconds()
),
new AdminFilesystemResponse.WebdavSection(false)
);
}
private String normalizeStorageProvider(String provider) {
if (!StringUtils.hasText(provider)) {
return "local";
}
return provider.trim().toLowerCase();
}
private long resolveEffectiveMaxFileSize(StoragePolicy policy, StoragePolicyCapabilities capabilities) {
long effectiveMaxFileSize = fileStorageProperties.getMaxFileSize();
if (policy.getMaxSizeBytes() > 0) {
effectiveMaxFileSize = Math.min(effectiveMaxFileSize, policy.getMaxSizeBytes());
}
if (capabilities.maxObjectSize() > 0) {
effectiveMaxFileSize = Math.min(effectiveMaxFileSize, capabilities.maxObjectSize());
}
return effectiveMaxFileSize;
}
private long readLongProperty(String key, long defaultValue) {
return environment.getProperty(key, Long.class, defaultValue);
}
}

View File

@@ -34,18 +34,47 @@ import java.util.List;
@PreAuthorize("@adminAccessEvaluator.isAdmin(authentication)") @PreAuthorize("@adminAccessEvaluator.isAdmin(authentication)")
public class AdminController { public class AdminController {
private final AdminService adminService; private final AdminInspectionQueryService adminInspectionQueryService;
private final AdminTaskQueryService adminTaskQueryService;
private final AdminStoragePolicyQueryService adminStoragePolicyQueryService;
private final AdminAuditQueryService adminAuditQueryService;
private final AdminResourceGovernanceService adminResourceGovernanceService;
private final AdminStorageGovernanceService adminStorageGovernanceService;
private final AdminConfigSnapshotService adminConfigSnapshotService;
private final AdminMutableSettingsService adminMutableSettingsService;
private final AdminUserGovernanceService adminUserGovernanceService;
private final CustomUserDetailsService userDetailsService; private final CustomUserDetailsService userDetailsService;
@GetMapping("/summary") @GetMapping("/summary")
public ApiResponse<AdminSummaryResponse> summary() { public ApiResponse<AdminSummaryResponse> summary() {
return ApiResponse.success(adminService.getSummary()); return ApiResponse.success(adminInspectionQueryService.getSummary());
}
@GetMapping("/settings")
public ApiResponse<AdminSettingsResponse> settings() {
return ApiResponse.success(adminConfigSnapshotService.getSettings());
}
@PatchMapping("/settings/registration/invite-code")
public ApiResponse<AdminRegistrationInviteCodeResponse> updateRegistrationInviteCode(
@Valid @RequestBody AdminRegistrationInviteCodeUpdateRequest request) {
return ApiResponse.success(adminMutableSettingsService.updateRegistrationInviteCode(request.inviteCode()));
}
@PostMapping("/settings/registration/invite-code/rotate")
public ApiResponse<AdminRegistrationInviteCodeResponse> rotateRegistrationInviteCode() {
return ApiResponse.success(adminMutableSettingsService.rotateRegistrationInviteCode());
}
@GetMapping("/filesystem")
public ApiResponse<AdminFilesystemResponse> filesystem() {
return ApiResponse.success(adminConfigSnapshotService.getFilesystem());
} }
@PatchMapping("/settings/offline-transfer-storage-limit") @PatchMapping("/settings/offline-transfer-storage-limit")
public ApiResponse<AdminOfflineTransferStorageLimitResponse> updateOfflineTransferStorageLimit( public ApiResponse<AdminOfflineTransferStorageLimitResponse> updateOfflineTransferStorageLimit(
@Valid @RequestBody AdminOfflineTransferStorageLimitUpdateRequest request) { @Valid @RequestBody AdminOfflineTransferStorageLimitUpdateRequest request) {
return ApiResponse.success(adminService.updateOfflineTransferStorageLimit( return ApiResponse.success(adminMutableSettingsService.updateOfflineTransferStorageLimit(
request.offlineTransferStorageLimitBytes() request.offlineTransferStorageLimitBytes()
)); ));
} }
@@ -54,7 +83,7 @@ public class AdminController {
public ApiResponse<PageResponse<AdminUserResponse>> users(@RequestParam(defaultValue = "0") int page, public ApiResponse<PageResponse<AdminUserResponse>> users(@RequestParam(defaultValue = "0") int page,
@RequestParam(defaultValue = "10") int size, @RequestParam(defaultValue = "10") int size,
@RequestParam(defaultValue = "") String query) { @RequestParam(defaultValue = "") String query) {
return ApiResponse.success(adminService.listUsers(page, size, query)); return ApiResponse.success(adminUserGovernanceService.listUsers(page, size, query));
} }
@GetMapping("/files") @GetMapping("/files")
@@ -62,7 +91,7 @@ public class AdminController {
@RequestParam(defaultValue = "10") int size, @RequestParam(defaultValue = "10") int size,
@RequestParam(defaultValue = "") String query, @RequestParam(defaultValue = "") String query,
@RequestParam(defaultValue = "") String ownerQuery) { @RequestParam(defaultValue = "") String ownerQuery) {
return ApiResponse.success(adminService.listFiles(page, size, query, ownerQuery)); return ApiResponse.success(adminInspectionQueryService.listFiles(page, size, query, ownerQuery));
} }
@GetMapping("/file-blobs") @GetMapping("/file-blobs")
@@ -72,7 +101,7 @@ public class AdminController {
@RequestParam(required = false) Long storagePolicyId, @RequestParam(required = false) Long storagePolicyId,
@RequestParam(defaultValue = "") String objectKey, @RequestParam(defaultValue = "") String objectKey,
@RequestParam(required = false) FileEntityType entityType) { @RequestParam(required = false) FileEntityType entityType) {
return ApiResponse.success(adminService.listFileBlobs(page, size, userQuery, storagePolicyId, objectKey, entityType)); return ApiResponse.success(adminInspectionQueryService.listFileBlobs(page, size, userQuery, storagePolicyId, objectKey, entityType));
} }
@GetMapping("/shares") @GetMapping("/shares")
@@ -83,12 +112,12 @@ public class AdminController {
@RequestParam(defaultValue = "") String token, @RequestParam(defaultValue = "") String token,
@RequestParam(required = false) Boolean passwordProtected, @RequestParam(required = false) Boolean passwordProtected,
@RequestParam(required = false) Boolean expired) { @RequestParam(required = false) Boolean expired) {
return ApiResponse.success(adminService.listShares(page, size, userQuery, fileName, token, passwordProtected, expired)); return ApiResponse.success(adminInspectionQueryService.listShares(page, size, userQuery, fileName, token, passwordProtected, expired));
} }
@DeleteMapping("/shares/{shareId}") @DeleteMapping("/shares/{shareId}")
public ApiResponse<Void> deleteShare(@PathVariable Long shareId) { public ApiResponse<Void> deleteShare(@PathVariable Long shareId) {
adminService.deleteShare(shareId); adminResourceGovernanceService.deleteShare(shareId);
return ApiResponse.success(); return ApiResponse.success();
} }
@@ -100,37 +129,54 @@ public class AdminController {
@RequestParam(required = false) BackgroundTaskStatus status, @RequestParam(required = false) BackgroundTaskStatus status,
@RequestParam(required = false) BackgroundTaskFailureCategory failureCategory, @RequestParam(required = false) BackgroundTaskFailureCategory failureCategory,
@RequestParam(required = false) AdminTaskLeaseState leaseState) { @RequestParam(required = false) AdminTaskLeaseState leaseState) {
return ApiResponse.success(adminService.listTasks(page, size, userQuery, type, status, failureCategory, leaseState)); return ApiResponse.success(adminTaskQueryService.listTasks(page, size, userQuery, type, status, failureCategory, leaseState));
} }
@GetMapping("/tasks/{taskId}") @GetMapping("/tasks/{taskId}")
public ApiResponse<AdminTaskResponse> task(@PathVariable Long taskId) { public ApiResponse<AdminTaskResponse> task(@PathVariable Long taskId) {
return ApiResponse.success(adminService.getTask(taskId)); return ApiResponse.success(adminTaskQueryService.getTask(taskId));
} }
@GetMapping("/storage-policies") @GetMapping("/storage-policies")
public ApiResponse<List<AdminStoragePolicyResponse>> storagePolicies() { public ApiResponse<List<AdminStoragePolicyResponse>> storagePolicies() {
return ApiResponse.success(adminService.listStoragePolicies()); return ApiResponse.success(adminStoragePolicyQueryService.listStoragePolicies());
}
@GetMapping("/audits")
public ApiResponse<PageResponse<AdminAuditLogResponse>> audits(@RequestParam(defaultValue = "0") int page,
@RequestParam(defaultValue = "10") int size,
@RequestParam(defaultValue = "") String actorQuery,
@RequestParam(defaultValue = "") String actionType,
@RequestParam(defaultValue = "") String targetType,
@RequestParam(required = false) Long targetId) {
return ApiResponse.success(adminAuditQueryService.listAuditLogs(
page,
size,
actorQuery,
actionType,
targetType,
targetId
));
} }
@PostMapping("/storage-policies") @PostMapping("/storage-policies")
public ApiResponse<AdminStoragePolicyResponse> createStoragePolicy( public ApiResponse<AdminStoragePolicyResponse> createStoragePolicy(
@Valid @RequestBody AdminStoragePolicyUpsertRequest request) { @Valid @RequestBody AdminStoragePolicyUpsertRequest request) {
return ApiResponse.success(adminService.createStoragePolicy(request)); return ApiResponse.success(adminStorageGovernanceService.createStoragePolicy(request));
} }
@PutMapping("/storage-policies/{policyId}") @PutMapping("/storage-policies/{policyId}")
public ApiResponse<AdminStoragePolicyResponse> updateStoragePolicy( public ApiResponse<AdminStoragePolicyResponse> updateStoragePolicy(
@PathVariable Long policyId, @PathVariable Long policyId,
@Valid @RequestBody AdminStoragePolicyUpsertRequest request) { @Valid @RequestBody AdminStoragePolicyUpsertRequest request) {
return ApiResponse.success(adminService.updateStoragePolicy(policyId, request)); return ApiResponse.success(adminStorageGovernanceService.updateStoragePolicy(policyId, request));
} }
@PatchMapping("/storage-policies/{policyId}/status") @PatchMapping("/storage-policies/{policyId}/status")
public ApiResponse<AdminStoragePolicyResponse> updateStoragePolicyStatus( public ApiResponse<AdminStoragePolicyResponse> updateStoragePolicyStatus(
@PathVariable Long policyId, @PathVariable Long policyId,
@Valid @RequestBody AdminStoragePolicyStatusUpdateRequest request) { @Valid @RequestBody AdminStoragePolicyStatusUpdateRequest request) {
return ApiResponse.success(adminService.updateStoragePolicyStatus(policyId, request.enabled())); return ApiResponse.success(adminStorageGovernanceService.updateStoragePolicyStatus(policyId, request.enabled()));
} }
@PostMapping("/storage-policies/migrations") @PostMapping("/storage-policies/migrations")
@@ -138,48 +184,48 @@ public class AdminController {
@AuthenticationPrincipal UserDetails userDetails, @AuthenticationPrincipal UserDetails userDetails,
@Valid @RequestBody AdminStoragePolicyMigrationCreateRequest request) { @Valid @RequestBody AdminStoragePolicyMigrationCreateRequest request) {
User user = userDetailsService.loadDomainUser(userDetails.getUsername()); User user = userDetailsService.loadDomainUser(userDetails.getUsername());
return ApiResponse.success(toTaskResponse(adminService.createStoragePolicyMigrationTask(user, request))); return ApiResponse.success(toTaskResponse(adminStorageGovernanceService.createStoragePolicyMigrationTask(user, request)));
} }
@DeleteMapping("/files/{fileId}") @DeleteMapping("/files/{fileId}")
public ApiResponse<Void> deleteFile(@PathVariable Long fileId) { public ApiResponse<Void> deleteFile(@PathVariable Long fileId) {
adminService.deleteFile(fileId); adminResourceGovernanceService.deleteFile(fileId);
return ApiResponse.success(); return ApiResponse.success();
} }
@PatchMapping("/users/{userId}/role") @PatchMapping("/users/{userId}/role")
public ApiResponse<AdminUserResponse> updateUserRole(@PathVariable Long userId, public ApiResponse<AdminUserResponse> updateUserRole(@PathVariable Long userId,
@Valid @RequestBody AdminUserRoleUpdateRequest request) { @Valid @RequestBody AdminUserRoleUpdateRequest request) {
return ApiResponse.success(adminService.updateUserRole(userId, request.role())); return ApiResponse.success(adminUserGovernanceService.updateUserRole(userId, request.role()));
} }
@PatchMapping("/users/{userId}/status") @PatchMapping("/users/{userId}/status")
public ApiResponse<AdminUserResponse> updateUserStatus(@PathVariable Long userId, public ApiResponse<AdminUserResponse> updateUserStatus(@PathVariable Long userId,
@Valid @RequestBody AdminUserStatusUpdateRequest request) { @Valid @RequestBody AdminUserStatusUpdateRequest request) {
return ApiResponse.success(adminService.updateUserBanned(userId, request.banned())); return ApiResponse.success(adminUserGovernanceService.updateUserBanned(userId, request.banned()));
} }
@PutMapping("/users/{userId}/password") @PutMapping("/users/{userId}/password")
public ApiResponse<AdminUserResponse> updateUserPassword(@PathVariable Long userId, public ApiResponse<AdminUserResponse> updateUserPassword(@PathVariable Long userId,
@Valid @RequestBody AdminUserPasswordUpdateRequest request) { @Valid @RequestBody AdminUserPasswordUpdateRequest request) {
return ApiResponse.success(adminService.updateUserPassword(userId, request.newPassword())); return ApiResponse.success(adminUserGovernanceService.updateUserPassword(userId, request.newPassword()));
} }
@PatchMapping("/users/{userId}/storage-quota") @PatchMapping("/users/{userId}/storage-quota")
public ApiResponse<AdminUserResponse> updateUserStorageQuota(@PathVariable Long userId, public ApiResponse<AdminUserResponse> updateUserStorageQuota(@PathVariable Long userId,
@Valid @RequestBody AdminUserStorageQuotaUpdateRequest request) { @Valid @RequestBody AdminUserStorageQuotaUpdateRequest request) {
return ApiResponse.success(adminService.updateUserStorageQuota(userId, request.storageQuotaBytes())); return ApiResponse.success(adminUserGovernanceService.updateUserStorageQuota(userId, request.storageQuotaBytes()));
} }
@PatchMapping("/users/{userId}/max-upload-size") @PatchMapping("/users/{userId}/max-upload-size")
public ApiResponse<AdminUserResponse> updateUserMaxUploadSize(@PathVariable Long userId, public ApiResponse<AdminUserResponse> updateUserMaxUploadSize(@PathVariable Long userId,
@Valid @RequestBody AdminUserMaxUploadSizeUpdateRequest request) { @Valid @RequestBody AdminUserMaxUploadSizeUpdateRequest request) {
return ApiResponse.success(adminService.updateUserMaxUploadSize(userId, request.maxUploadSizeBytes())); return ApiResponse.success(adminUserGovernanceService.updateUserMaxUploadSize(userId, request.maxUploadSizeBytes()));
} }
@PostMapping("/users/{userId}/password/reset") @PostMapping("/users/{userId}/password/reset")
public ApiResponse<AdminPasswordResetResponse> resetUserPassword(@PathVariable Long userId) { public ApiResponse<AdminPasswordResetResponse> resetUserPassword(@PathVariable Long userId) {
return ApiResponse.success(adminService.resetUserPassword(userId)); return ApiResponse.success(adminUserGovernanceService.resetUserPassword(userId));
} }
private BackgroundTaskResponse toTaskResponse(BackgroundTask task) { private BackgroundTaskResponse toTaskResponse(BackgroundTask task) {

View File

@@ -0,0 +1,45 @@
package com.yoyuzh.admin;
public record AdminFilesystemResponse(
OverviewSection overview,
AdminStoragePolicyResponse defaultPolicy,
UploadSection upload,
MediaProcessingSection mediaProcessing,
CacheSection cache,
WebdavSection webdav
) {
public record OverviewSection(
String storageProvider,
long totalFiles,
long totalBlobs,
long totalEntities
) {
}
public record UploadSection(
boolean proxyUpload,
boolean directSingleUpload,
boolean directMultipartUpload,
long effectiveMaxFileSizeBytes
) {
}
public record MediaProcessingSection(
boolean metadataExtractionEnabled,
boolean nativeThumbnailSupport
) {
}
public record CacheSection(
String backend,
long filesListTtlSeconds,
long directoryVersionTtlSeconds
) {
}
public record WebdavSection(
boolean enabled
) {
}
}

View File

@@ -0,0 +1,242 @@
package com.yoyuzh.admin;
import com.yoyuzh.auth.RegistrationInviteService;
import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.core.FileEntity;
import com.yoyuzh.files.core.FileEntityRepository;
import com.yoyuzh.files.core.FileEntityType;
import com.yoyuzh.files.core.FileBlob;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileEntityRepository;
import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.share.FileShareLink;
import com.yoyuzh.files.share.FileShareLinkRepository;
import com.yoyuzh.transfer.OfflineTransferSessionRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class AdminInspectionQueryService {
private final UserRepository userRepository;
private final StoredFileRepository storedFileRepository;
private final FileBlobRepository fileBlobRepository;
private final RegistrationInviteService registrationInviteService;
private final OfflineTransferSessionRepository offlineTransferSessionRepository;
private final AdminMetricsService adminMetricsService;
private final FileEntityRepository fileEntityRepository;
private final StoredFileEntityRepository storedFileEntityRepository;
private final FileShareLinkRepository fileShareLinkRepository;
public AdminSummaryResponse getSummary() {
AdminMetricsSnapshot metrics = adminMetricsService.getSnapshot();
return new AdminSummaryResponse(
userRepository.count(),
storedFileRepository.count(),
fileBlobRepository.sumAllBlobSize(),
metrics.downloadTrafficBytes(),
metrics.requestCount(),
metrics.transferUsageBytes(),
offlineTransferSessionRepository.sumUploadedFileSizeByExpiresAtAfter(Instant.now()),
metrics.offlineTransferStorageLimitBytes(),
metrics.dailyActiveUsers(),
metrics.requestTimeline(),
registrationInviteService.getCurrentInviteCode()
);
}
public PageResponse<AdminFileResponse> listFiles(int page, int size, String query, String ownerQuery) {
Page<StoredFile> result = storedFileRepository.searchAdminFiles(
normalizeQuery(query),
normalizeQuery(ownerQuery),
PageRequest.of(page, size, Sort.by(Sort.Direction.ASC, "user.username")
.and(Sort.by(Sort.Direction.DESC, "createdAt")))
);
List<AdminFileResponse> items = result.getContent().stream()
.map(this::toFileResponse)
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
public PageResponse<AdminFileBlobResponse> listFileBlobs(int page,
int size,
String userQuery,
Long storagePolicyId,
String objectKey,
FileEntityType entityType) {
Page<FileEntity> result = fileEntityRepository.searchAdminEntities(
normalizeQuery(userQuery),
storagePolicyId,
normalizeQuery(objectKey),
entityType,
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
List<FileEntity> entities = result.getContent();
Map<String, FileBlob> blobsByObjectKey = loadBlobsByObjectKey(entities);
Map<Long, StoredFileEntityRepository.FileEntityLinkStatsProjection> linkStatsByEntityId = loadLinkStatsByEntityId(entities);
List<AdminFileBlobResponse> items = entities.stream()
.map(entity -> toFileBlobResponse(
entity,
blobsByObjectKey.get(entity.getObjectKey()),
linkStatsByEntityId.get(entity.getId())
))
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
public PageResponse<AdminShareResponse> listShares(int page,
int size,
String userQuery,
String fileName,
String token,
Boolean passwordProtected,
Boolean expired) {
Page<FileShareLink> result = fileShareLinkRepository.searchAdminShares(
normalizeQuery(userQuery),
normalizeQuery(fileName),
normalizeQuery(token),
passwordProtected,
expired,
LocalDateTime.now(),
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
List<AdminShareResponse> items = result.getContent().stream()
.map(this::toAdminShareResponse)
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
private AdminFileResponse toFileResponse(StoredFile storedFile) {
User owner = storedFile.getUser();
return new AdminFileResponse(
storedFile.getId(),
storedFile.getFilename(),
storedFile.getPath(),
storedFile.getSize(),
storedFile.getContentType(),
storedFile.isDirectory(),
storedFile.getCreatedAt(),
owner.getId(),
owner.getUsername(),
owner.getEmail()
);
}
private AdminFileBlobResponse toFileBlobResponse(FileEntity entity,
FileBlob blob,
StoredFileEntityRepository.FileEntityLinkStatsProjection linkStats) {
long linkedStoredFileCount = linkStats == null || linkStats.getLinkedStoredFileCount() == null
? 0L
: linkStats.getLinkedStoredFileCount();
long linkedOwnerCount = linkStats == null || linkStats.getLinkedOwnerCount() == null
? 0L
: linkStats.getLinkedOwnerCount();
String sampleOwnerUsername = linkStats == null ? null : linkStats.getSampleOwnerUsername();
String sampleOwnerEmail = linkStats == null ? null : linkStats.getSampleOwnerEmail();
return new AdminFileBlobResponse(
entity.getId(),
blob == null ? null : blob.getId(),
entity.getObjectKey(),
entity.getEntityType(),
entity.getStoragePolicyId(),
entity.getSize(),
StringUtils.hasText(entity.getContentType()) ? entity.getContentType() : blob == null ? null : blob.getContentType(),
entity.getReferenceCount(),
linkedStoredFileCount,
linkedOwnerCount,
sampleOwnerUsername,
sampleOwnerEmail,
entity.getCreatedBy() == null ? null : entity.getCreatedBy().getId(),
entity.getCreatedBy() == null ? null : entity.getCreatedBy().getUsername(),
entity.getCreatedAt(),
blob == null ? null : blob.getCreatedAt(),
blob == null,
linkedStoredFileCount == 0,
entity.getReferenceCount() == null || entity.getReferenceCount() != linkedStoredFileCount
);
}
private Map<String, FileBlob> loadBlobsByObjectKey(List<FileEntity> entities) {
Set<String> objectKeys = entities.stream()
.map(FileEntity::getObjectKey)
.filter(StringUtils::hasText)
.collect(Collectors.toSet());
if (objectKeys.isEmpty()) {
return Map.of();
}
return fileBlobRepository.findAllByObjectKeyIn(objectKeys).stream()
.collect(Collectors.toMap(
FileBlob::getObjectKey,
Function.identity(),
(left, right) -> left
));
}
private Map<Long, StoredFileEntityRepository.FileEntityLinkStatsProjection> loadLinkStatsByEntityId(List<FileEntity> entities) {
Set<Long> entityIds = entities.stream()
.map(FileEntity::getId)
.filter(id -> id != null)
.collect(Collectors.toSet());
if (entityIds.isEmpty()) {
return Collections.emptyMap();
}
return storedFileEntityRepository.findAdminLinkStatsByFileEntityIds(entityIds).stream()
.collect(Collectors.toMap(
StoredFileEntityRepository.FileEntityLinkStatsProjection::getFileEntityId,
Function.identity()
));
}
private AdminShareResponse toAdminShareResponse(FileShareLink shareLink) {
StoredFile file = shareLink.getFile();
User owner = shareLink.getOwner();
boolean expired = shareLink.getExpiresAt() != null && shareLink.getExpiresAt().isBefore(LocalDateTime.now());
return new AdminShareResponse(
shareLink.getId(),
shareLink.getToken(),
shareLink.getShareNameOrDefault(),
shareLink.hasPassword(),
expired,
shareLink.getCreatedAt(),
shareLink.getExpiresAt(),
shareLink.getMaxDownloads(),
shareLink.getDownloadCountOrZero(),
shareLink.getViewCountOrZero(),
shareLink.isAllowImportEnabled(),
shareLink.isAllowDownloadEnabled(),
owner.getId(),
owner.getUsername(),
owner.getEmail(),
file.getId(),
file.getFilename(),
file.getPath(),
file.getContentType(),
file.getSize(),
file.isDirectory()
);
}
private String normalizeQuery(String query) {
if (query == null) {
return "";
}
return query.trim();
}
}

View File

@@ -0,0 +1,66 @@
package com.yoyuzh.admin;
import com.yoyuzh.auth.RegistrationInviteService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class AdminMutableSettingsService {
private final RegistrationInviteService registrationInviteService;
private final AdminMetricsService adminMetricsService;
private final AdminAuditService adminAuditService;
@Transactional
public AdminRegistrationInviteCodeResponse updateRegistrationInviteCode(String inviteCode) {
String normalizedInviteCode = normalizeQuery(inviteCode);
String currentInviteCode = registrationInviteService.updateCurrentInviteCode(normalizedInviteCode);
adminAuditService.record(
AdminAuditAction.UPDATE_REGISTRATION_INVITE_CODE,
"SYSTEM_SETTING",
null,
"Updated registration invite code",
Map.of("inviteCodeLength", currentInviteCode.length())
);
return new AdminRegistrationInviteCodeResponse(currentInviteCode);
}
@Transactional
public AdminRegistrationInviteCodeResponse rotateRegistrationInviteCode() {
String currentInviteCode = registrationInviteService.rotateCurrentInviteCode();
adminAuditService.record(
AdminAuditAction.ROTATE_REGISTRATION_INVITE_CODE,
"SYSTEM_SETTING",
null,
"Rotated registration invite code",
Map.of("inviteCodeLength", currentInviteCode.length())
);
return new AdminRegistrationInviteCodeResponse(currentInviteCode);
}
@Transactional
public AdminOfflineTransferStorageLimitResponse updateOfflineTransferStorageLimit(long offlineTransferStorageLimitBytes) {
AdminOfflineTransferStorageLimitResponse response = adminMetricsService.updateOfflineTransferStorageLimit(
offlineTransferStorageLimitBytes
);
adminAuditService.record(
AdminAuditAction.UPDATE_OFFLINE_TRANSFER_STORAGE_LIMIT,
"SYSTEM_SETTING",
null,
"Updated offline transfer storage limit",
Map.of("offlineTransferStorageLimitBytes", response.offlineTransferStorageLimitBytes())
);
return response;
}
private String normalizeQuery(String query) {
if (query == null) {
return "";
}
return query.trim();
}
}

View File

@@ -0,0 +1,6 @@
package com.yoyuzh.admin;
public record AdminRegistrationInviteCodeResponse(
String currentInviteCode
) {
}

View File

@@ -0,0 +1,11 @@
package com.yoyuzh.admin;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Size;
public record AdminRegistrationInviteCodeUpdateRequest(
@NotBlank(message = "邀请码不能为空")
@Size(max = 64, message = "邀请码长度不能超过 64 个字符")
String inviteCode
) {
}

View File

@@ -0,0 +1,60 @@
package com.yoyuzh.admin;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.share.FileShareLink;
import com.yoyuzh.files.share.FileShareLinkRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.LinkedHashMap;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class AdminResourceGovernanceService {
private final StoredFileRepository storedFileRepository;
private final FileService fileService;
private final FileShareLinkRepository fileShareLinkRepository;
private final AdminAuditService adminAuditService;
@Transactional
public void deleteShare(Long shareId) {
FileShareLink shareLink = fileShareLinkRepository.findById(shareId)
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "share not found"));
fileShareLinkRepository.delete(shareLink);
Map<String, Object> details = new LinkedHashMap<>();
details.put("token", shareLink.getToken());
adminAuditService.record(
AdminAuditAction.DELETE_SHARE,
"SHARE",
shareId,
"Deleted share link",
details
);
}
@Transactional
public void deleteFile(Long fileId) {
StoredFile storedFile = storedFileRepository.findById(fileId)
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "file not found"));
fileService.delete(storedFile.getUser(), fileId);
Map<String, Object> details = new LinkedHashMap<>();
details.put("ownerUserId", storedFile.getUser().getId());
details.put("path", storedFile.getPath());
details.put("filename", storedFile.getFilename());
details.put("directory", storedFile.isDirectory());
adminAuditService.record(
AdminAuditAction.DELETE_FILE,
"FILE",
fileId,
"Deleted file",
details
);
}
}

View File

@@ -1,611 +0,0 @@
package com.yoyuzh.admin;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.auth.AuthTokenInvalidationService;
import com.yoyuzh.auth.PasswordPolicy;
import com.yoyuzh.auth.RefreshTokenService;
import com.yoyuzh.auth.RegistrationInviteService;
import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.auth.UserRole;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.common.PageResponse;
import com.yoyuzh.config.RedisCacheNames;
import com.yoyuzh.files.core.FileEntity;
import com.yoyuzh.files.core.FileEntityRepository;
import com.yoyuzh.files.core.FileEntityType;
import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileEntityRepository;
import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyRepository;
import com.yoyuzh.files.policy.StoragePolicyService;
import com.yoyuzh.files.share.FileShareLink;
import com.yoyuzh.files.share.FileShareLinkRepository;
import com.yoyuzh.files.tasks.BackgroundTask;
import com.yoyuzh.files.tasks.BackgroundTaskFailureCategory;
import com.yoyuzh.files.tasks.BackgroundTaskRepository;
import com.yoyuzh.files.tasks.BackgroundTaskService;
import com.yoyuzh.files.tasks.BackgroundTaskStatus;
import com.yoyuzh.files.tasks.BackgroundTaskType;
import com.yoyuzh.transfer.OfflineTransferSessionRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import java.security.SecureRandom;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class AdminService {
private final UserRepository userRepository;
private final StoredFileRepository storedFileRepository;
private final FileBlobRepository fileBlobRepository;
private final FileService fileService;
private final PasswordEncoder passwordEncoder;
private final RefreshTokenService refreshTokenService;
private final AuthTokenInvalidationService authTokenInvalidationService;
private final RegistrationInviteService registrationInviteService;
private final OfflineTransferSessionRepository offlineTransferSessionRepository;
private final AdminMetricsService adminMetricsService;
private final StoragePolicyRepository storagePolicyRepository;
private final StoragePolicyService storagePolicyService;
private final FileEntityRepository fileEntityRepository;
private final StoredFileEntityRepository storedFileEntityRepository;
private final BackgroundTaskRepository backgroundTaskRepository;
private final BackgroundTaskService backgroundTaskService;
private final FileShareLinkRepository fileShareLinkRepository;
private final ObjectMapper objectMapper;
private final SecureRandom secureRandom = new SecureRandom();
public AdminSummaryResponse getSummary() {
AdminMetricsSnapshot metrics = adminMetricsService.getSnapshot();
return new AdminSummaryResponse(
userRepository.count(),
storedFileRepository.count(),
fileBlobRepository.sumAllBlobSize(),
metrics.downloadTrafficBytes(),
metrics.requestCount(),
metrics.transferUsageBytes(),
offlineTransferSessionRepository.sumUploadedFileSizeByExpiresAtAfter(Instant.now()),
metrics.offlineTransferStorageLimitBytes(),
metrics.dailyActiveUsers(),
metrics.requestTimeline(),
registrationInviteService.getCurrentInviteCode()
);
}
public PageResponse<AdminUserResponse> listUsers(int page, int size, String query) {
Page<User> result = userRepository.searchByUsernameOrEmail(
normalizeQuery(query),
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
List<AdminUserResponse> items = result.getContent().stream()
.map(this::toUserResponse)
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
public PageResponse<AdminFileResponse> listFiles(int page, int size, String query, String ownerQuery) {
Page<StoredFile> result = storedFileRepository.searchAdminFiles(
normalizeQuery(query),
normalizeQuery(ownerQuery),
PageRequest.of(page, size, Sort.by(Sort.Direction.ASC, "user.username")
.and(Sort.by(Sort.Direction.DESC, "createdAt")))
);
List<AdminFileResponse> items = result.getContent().stream()
.map(this::toFileResponse)
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
public PageResponse<AdminFileBlobResponse> listFileBlobs(int page,
int size,
String userQuery,
Long storagePolicyId,
String objectKey,
FileEntityType entityType) {
Page<FileEntity> result = fileEntityRepository.searchAdminEntities(
normalizeQuery(userQuery),
storagePolicyId,
normalizeQuery(objectKey),
entityType,
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
List<AdminFileBlobResponse> items = result.getContent().stream()
.map(this::toFileBlobResponse)
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
public PageResponse<AdminShareResponse> listShares(int page,
int size,
String userQuery,
String fileName,
String token,
Boolean passwordProtected,
Boolean expired) {
Page<FileShareLink> result = fileShareLinkRepository.searchAdminShares(
normalizeQuery(userQuery),
normalizeQuery(fileName),
normalizeQuery(token),
passwordProtected,
expired,
LocalDateTime.now(),
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
List<AdminShareResponse> items = result.getContent().stream()
.map(this::toAdminShareResponse)
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
@Transactional
public void deleteShare(Long shareId) {
FileShareLink shareLink = fileShareLinkRepository.findById(shareId)
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "share not found"));
fileShareLinkRepository.delete(shareLink);
}
public PageResponse<AdminTaskResponse> listTasks(int page,
int size,
String userQuery,
BackgroundTaskType type,
BackgroundTaskStatus status,
BackgroundTaskFailureCategory failureCategory,
AdminTaskLeaseState leaseState) {
String failureCategoryPattern = failureCategory == null
? null
: "\"failureCategory\":\"" + failureCategory.name() + "\"";
Page<BackgroundTask> result = backgroundTaskRepository.searchAdminTasks(
normalizeQuery(userQuery),
type,
status,
failureCategoryPattern,
leaseState == null ? null : leaseState.name(),
LocalDateTime.now(),
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
Map<Long, User> ownerById = userRepository.findAllById(result.getContent().stream()
.map(BackgroundTask::getUserId)
.collect(Collectors.toSet()))
.stream()
.collect(Collectors.toMap(User::getId, user -> user));
List<AdminTaskResponse> items = result.getContent().stream()
.map(task -> toAdminTaskResponse(task, ownerById.get(task.getUserId())))
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
public AdminTaskResponse getTask(Long taskId) {
BackgroundTask task = backgroundTaskRepository.findById(taskId)
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "task not found"));
User owner = userRepository.findById(task.getUserId()).orElse(null);
return toAdminTaskResponse(task, owner);
}
@Cacheable(cacheNames = RedisCacheNames.STORAGE_POLICIES, key = "'all'")
public List<AdminStoragePolicyResponse> listStoragePolicies() {
return storagePolicyRepository.findAll(Sort.by(Sort.Direction.DESC, "defaultPolicy")
.and(Sort.by(Sort.Direction.DESC, "enabled"))
.and(Sort.by(Sort.Direction.ASC, "id")))
.stream()
.map(this::toStoragePolicyResponse)
.toList();
}
@Transactional
@CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true)
public AdminStoragePolicyResponse createStoragePolicy(AdminStoragePolicyUpsertRequest request) {
StoragePolicy policy = new StoragePolicy();
policy.setDefaultPolicy(false);
applyStoragePolicyUpsert(policy, request);
return toStoragePolicyResponse(storagePolicyRepository.save(policy));
}
@Transactional
@CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true)
public AdminStoragePolicyResponse updateStoragePolicy(Long policyId, AdminStoragePolicyUpsertRequest request) {
StoragePolicy policy = getRequiredStoragePolicy(policyId);
applyStoragePolicyUpsert(policy, request);
return toStoragePolicyResponse(storagePolicyRepository.save(policy));
}
@Transactional
@CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true)
public AdminStoragePolicyResponse updateStoragePolicyStatus(Long policyId, boolean enabled) {
StoragePolicy policy = getRequiredStoragePolicy(policyId);
if (policy.isDefaultPolicy() && !enabled) {
throw new BusinessException(ErrorCode.UNKNOWN, "榛樿瀛樺偍绛栫暐涓嶈兘鍋滅敤");
}
policy.setEnabled(enabled);
return toStoragePolicyResponse(storagePolicyRepository.save(policy));
}
@Transactional
public BackgroundTask createStoragePolicyMigrationTask(User user, AdminStoragePolicyMigrationCreateRequest request) {
StoragePolicy sourcePolicy = getRequiredStoragePolicy(request.sourcePolicyId());
StoragePolicy targetPolicy = getRequiredStoragePolicy(request.targetPolicyId());
if (sourcePolicy.getId().equals(targetPolicy.getId())) {
throw new BusinessException(ErrorCode.UNKNOWN, "婧愬瓨鍌ㄧ瓥鐣ュ拰鐩爣瀛樺偍绛栫暐涓嶈兘鐩稿悓");
}
if (!targetPolicy.isEnabled()) {
throw new BusinessException(ErrorCode.UNKNOWN, "target storage policy must be enabled");
}
long candidateEntityCount = fileEntityRepository.countByStoragePolicyIdAndEntityType(
sourcePolicy.getId(),
FileEntityType.VERSION
);
long candidateStoredFileCount = storedFileEntityRepository.countDistinctStoredFilesByStoragePolicyIdAndEntityType(
sourcePolicy.getId(),
FileEntityType.VERSION
);
Map<String, Object> state = new LinkedHashMap<>();
state.put("sourcePolicyId", sourcePolicy.getId());
state.put("sourcePolicyName", sourcePolicy.getName());
state.put("targetPolicyId", targetPolicy.getId());
state.put("targetPolicyName", targetPolicy.getName());
state.put("candidateEntityCount", candidateEntityCount);
state.put("candidateStoredFileCount", candidateStoredFileCount);
state.put("migrationPerformed", false);
state.put("migrationMode", "skeleton");
state.put("entityType", FileEntityType.VERSION.name());
state.put("message", "storage policy migration skeleton queued; worker will validate and recount candidates without moving object data");
Map<String, Object> privateState = new LinkedHashMap<>(state);
privateState.put("taskType", BackgroundTaskType.STORAGE_POLICY_MIGRATION.name());
return backgroundTaskService.createQueuedTask(
user,
BackgroundTaskType.STORAGE_POLICY_MIGRATION,
state,
privateState,
request.correlationId()
);
}
@Transactional
public void deleteFile(Long fileId) {
StoredFile storedFile = storedFileRepository.findById(fileId)
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "file not found"));
fileService.delete(storedFile.getUser(), fileId);
}
@Transactional
public AdminUserResponse updateUserRole(Long userId, UserRole role) {
User user = getRequiredUser(userId);
user.setRole(role);
return toUserResponse(userRepository.save(user));
}
@Transactional
public AdminUserResponse updateUserBanned(Long userId, boolean banned) {
User user = getRequiredUser(userId);
user.setBanned(banned);
authTokenInvalidationService.revokeAccessTokensForUser(user.getId());
user.setActiveSessionId(UUID.randomUUID().toString());
user.setDesktopActiveSessionId(UUID.randomUUID().toString());
user.setMobileActiveSessionId(UUID.randomUUID().toString());
refreshTokenService.revokeAllForUser(user.getId());
return toUserResponse(userRepository.save(user));
}
@Transactional
public AdminUserResponse updateUserPassword(Long userId, String newPassword) {
if (!PasswordPolicy.isStrong(newPassword)) {
throw new BusinessException(ErrorCode.UNKNOWN, PasswordPolicy.VALIDATION_MESSAGE);
}
User user = getRequiredUser(userId);
user.setPasswordHash(passwordEncoder.encode(newPassword));
authTokenInvalidationService.revokeAccessTokensForUser(user.getId());
user.setActiveSessionId(UUID.randomUUID().toString());
user.setDesktopActiveSessionId(UUID.randomUUID().toString());
user.setMobileActiveSessionId(UUID.randomUUID().toString());
refreshTokenService.revokeAllForUser(user.getId());
return toUserResponse(userRepository.save(user));
}
@Transactional
public AdminUserResponse updateUserStorageQuota(Long userId, long storageQuotaBytes) {
User user = getRequiredUser(userId);
user.setStorageQuotaBytes(storageQuotaBytes);
return toUserResponse(userRepository.save(user));
}
@Transactional
public AdminUserResponse updateUserMaxUploadSize(Long userId, long maxUploadSizeBytes) {
User user = getRequiredUser(userId);
user.setMaxUploadSizeBytes(maxUploadSizeBytes);
return toUserResponse(userRepository.save(user));
}
@Transactional
public AdminPasswordResetResponse resetUserPassword(Long userId) {
String temporaryPassword = generateTemporaryPassword();
updateUserPassword(userId, temporaryPassword);
return new AdminPasswordResetResponse(temporaryPassword);
}
@Transactional
public AdminOfflineTransferStorageLimitResponse updateOfflineTransferStorageLimit(long offlineTransferStorageLimitBytes) {
return adminMetricsService.updateOfflineTransferStorageLimit(offlineTransferStorageLimitBytes);
}
private AdminUserResponse toUserResponse(User user) {
long usedStorageBytes = storedFileRepository.sumFileSizeByUserId(user.getId());
return new AdminUserResponse(
user.getId(),
user.getUsername(),
user.getEmail(),
user.getPhoneNumber(),
user.getCreatedAt(),
user.getRole(),
user.isBanned(),
usedStorageBytes,
user.getStorageQuotaBytes(),
user.getMaxUploadSizeBytes()
);
}
private AdminFileResponse toFileResponse(StoredFile storedFile) {
User owner = storedFile.getUser();
return new AdminFileResponse(
storedFile.getId(),
storedFile.getFilename(),
storedFile.getPath(),
storedFile.getSize(),
storedFile.getContentType(),
storedFile.isDirectory(),
storedFile.getCreatedAt(),
owner.getId(),
owner.getUsername(),
owner.getEmail()
);
}
private AdminStoragePolicyResponse toStoragePolicyResponse(StoragePolicy policy) {
return new AdminStoragePolicyResponse(
policy.getId(),
policy.getName(),
policy.getType(),
policy.getBucketName(),
policy.getEndpoint(),
policy.getRegion(),
policy.isPrivateBucket(),
policy.getPrefix(),
policy.getCredentialMode(),
policy.getMaxSizeBytes(),
storagePolicyService.readCapabilities(policy),
policy.isEnabled(),
policy.isDefaultPolicy(),
policy.getCreatedAt(),
policy.getUpdatedAt()
);
}
private AdminFileBlobResponse toFileBlobResponse(FileEntity entity) {
var blob = fileBlobRepository.findByObjectKey(entity.getObjectKey()).orElse(null);
long linkedStoredFileCount = storedFileEntityRepository.countByFileEntityId(entity.getId());
long linkedOwnerCount = storedFileEntityRepository.countDistinctOwnersByFileEntityId(entity.getId());
return new AdminFileBlobResponse(
entity.getId(),
blob == null ? null : blob.getId(),
entity.getObjectKey(),
entity.getEntityType(),
entity.getStoragePolicyId(),
entity.getSize(),
StringUtils.hasText(entity.getContentType()) ? entity.getContentType() : blob == null ? null : blob.getContentType(),
entity.getReferenceCount(),
linkedStoredFileCount,
linkedOwnerCount,
storedFileEntityRepository.findSampleOwnerUsernameByFileEntityId(entity.getId()),
storedFileEntityRepository.findSampleOwnerEmailByFileEntityId(entity.getId()),
entity.getCreatedBy() == null ? null : entity.getCreatedBy().getId(),
entity.getCreatedBy() == null ? null : entity.getCreatedBy().getUsername(),
entity.getCreatedAt(),
blob == null ? null : blob.getCreatedAt(),
blob == null,
linkedStoredFileCount == 0,
entity.getReferenceCount() == null || entity.getReferenceCount() != linkedStoredFileCount
);
}
private AdminShareResponse toAdminShareResponse(FileShareLink shareLink) {
StoredFile file = shareLink.getFile();
User owner = shareLink.getOwner();
boolean expired = shareLink.getExpiresAt() != null && shareLink.getExpiresAt().isBefore(LocalDateTime.now());
return new AdminShareResponse(
shareLink.getId(),
shareLink.getToken(),
shareLink.getShareNameOrDefault(),
shareLink.hasPassword(),
expired,
shareLink.getCreatedAt(),
shareLink.getExpiresAt(),
shareLink.getMaxDownloads(),
shareLink.getDownloadCountOrZero(),
shareLink.getViewCountOrZero(),
shareLink.isAllowImportEnabled(),
shareLink.isAllowDownloadEnabled(),
owner.getId(),
owner.getUsername(),
owner.getEmail(),
file.getId(),
file.getFilename(),
file.getPath(),
file.getContentType(),
file.getSize(),
file.isDirectory()
);
}
private AdminTaskResponse toAdminTaskResponse(BackgroundTask task, User owner) {
Map<String, Object> state = parseState(task.getPublicStateJson());
return new AdminTaskResponse(
task.getId(),
task.getType(),
task.getStatus(),
task.getUserId(),
owner == null ? null : owner.getUsername(),
owner == null ? null : owner.getEmail(),
task.getPublicStateJson(),
task.getCorrelationId(),
task.getErrorMessage(),
task.getAttemptCount(),
task.getMaxAttempts(),
task.getNextRunAt(),
task.getLeaseOwner(),
task.getLeaseExpiresAt(),
task.getHeartbeatAt(),
task.getCreatedAt(),
task.getUpdatedAt(),
task.getFinishedAt(),
readStringState(state, "failureCategory"),
readBooleanState(state, "retryScheduled"),
readStringState(state, "workerOwner"),
resolveLeaseState(task)
);
}
private void applyStoragePolicyUpsert(StoragePolicy policy, AdminStoragePolicyUpsertRequest request) {
if (policy.isDefaultPolicy() && !request.enabled()) {
throw new BusinessException(ErrorCode.UNKNOWN, "榛樿瀛樺偍绛栫暐涓嶈兘鍋滅敤");
}
validateStoragePolicyRequest(request);
policy.setName(request.name().trim());
policy.setType(request.type());
policy.setBucketName(normalizeNullable(request.bucketName()));
policy.setEndpoint(normalizeNullable(request.endpoint()));
policy.setRegion(normalizeNullable(request.region()));
policy.setPrivateBucket(request.privateBucket());
policy.setPrefix(normalizePrefix(request.prefix()));
policy.setCredentialMode(request.credentialMode());
policy.setMaxSizeBytes(request.maxSizeBytes());
policy.setCapabilitiesJson(storagePolicyService.writeCapabilities(request.capabilities()));
policy.setEnabled(request.enabled());
}
private User getRequiredUser(Long userId) {
return userRepository.findById(userId)
.orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "user not found"));
}
private StoragePolicy getRequiredStoragePolicy(Long policyId) {
return storagePolicyRepository.findById(policyId)
.orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "storage policy not found"));
}
private String normalizeQuery(String query) {
if (query == null) {
return "";
}
return query.trim();
}
private String normalizeNullable(String value) {
if (!StringUtils.hasText(value)) {
return null;
}
return value.trim();
}
private String normalizePrefix(String prefix) {
if (!StringUtils.hasText(prefix)) {
return "";
}
return prefix.trim();
}
private Map<String, Object> parseState(String json) {
if (!StringUtils.hasText(json)) {
return Map.of();
}
try {
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
return Map.of();
}
}
private String readStringState(Map<String, Object> state, String key) {
Object value = state.get(key);
return value == null ? null : String.valueOf(value);
}
private Boolean readBooleanState(Map<String, Object> state, String key) {
Object value = state.get(key);
if (value instanceof Boolean boolValue) {
return boolValue;
}
if (value instanceof String stringValue) {
return Boolean.parseBoolean(stringValue);
}
return null;
}
private AdminTaskLeaseState resolveLeaseState(BackgroundTask task) {
if (!StringUtils.hasText(task.getLeaseOwner()) || task.getLeaseExpiresAt() == null) {
return AdminTaskLeaseState.NONE;
}
return task.getLeaseExpiresAt().isBefore(LocalDateTime.now())
? AdminTaskLeaseState.EXPIRED
: AdminTaskLeaseState.ACTIVE;
}
private void validateStoragePolicyRequest(AdminStoragePolicyUpsertRequest request) {
if (request.type() == com.yoyuzh.files.policy.StoragePolicyType.LOCAL
&& request.credentialMode() != com.yoyuzh.files.policy.StoragePolicyCredentialMode.NONE) {
throw new BusinessException(ErrorCode.UNKNOWN, "鏈湴瀛樺偍绛栫暐蹇呴』浣跨敤 NONE 鍑瘉妯″紡");
}
if (request.type() == com.yoyuzh.files.policy.StoragePolicyType.S3_COMPATIBLE
&& !StringUtils.hasText(request.bucketName())) {
throw new BusinessException(ErrorCode.UNKNOWN, "S3 瀛樺偍绛栫暐蹇呴』鎻愪緵 bucketName");
}
}
private String generateTemporaryPassword() {
String lowers = "abcdefghjkmnpqrstuvwxyz";
String uppers = "ABCDEFGHJKMNPQRSTUVWXYZ";
String digits = "23456789";
String specials = "!@#$%^&*";
String all = lowers + uppers + digits + specials;
char[] password = new char[12];
password[0] = lowers.charAt(secureRandom.nextInt(lowers.length()));
password[1] = uppers.charAt(secureRandom.nextInt(uppers.length()));
password[2] = digits.charAt(secureRandom.nextInt(digits.length()));
password[3] = specials.charAt(secureRandom.nextInt(specials.length()));
for (int i = 4; i < password.length; i += 1) {
password[i] = all.charAt(secureRandom.nextInt(all.length()));
}
for (int i = password.length - 1; i > 0; i -= 1) {
int j = secureRandom.nextInt(i + 1);
char tmp = password[i];
password[i] = password[j];
password[j] = tmp;
}
return new String(password);
}
}

View File

@@ -0,0 +1,73 @@
package com.yoyuzh.admin;
import java.util.List;
public record AdminSettingsResponse(
SiteSection site,
RegistrationSection registration,
UserSessionSection userSession,
TransferSection transfer,
MediaProcessingSection mediaProcessing,
QueueSection queue,
AppearanceSection appearance,
ServerSection server
) {
public record SiteSection(
boolean supported,
boolean writeSupported
) {
}
public record RegistrationSection(
boolean inviteCodeRequired,
String currentInviteCode,
List<String> managementRoles,
boolean writeSupported
) {
}
public record UserSessionSection(
long accessExpirationSeconds,
long refreshExpirationSeconds,
boolean tokenBlacklistEnabled,
long tokenBlacklistTtlBufferSeconds,
boolean writeSupported
) {
}
public record TransferSection(
long offlineTransferStorageLimitBytes,
boolean writeSupported
) {
}
public record MediaProcessingSection(
boolean metadataExtractionEnabled,
boolean thumbnailGenerationEnabled,
boolean videoPosterEnabled,
boolean writeSupported
) {
}
public record QueueSection(
String backend,
long mediaMetadataFixedDelayMs,
long mediaMetadataInitialDelayMs,
boolean writeSupported
) {
}
public record AppearanceSection(
boolean supported,
boolean writeSupported
) {
}
public record ServerSection(
String storageProvider,
boolean redisEnabled,
boolean writeSupported
) {
}
}

View File

@@ -0,0 +1,199 @@
package com.yoyuzh.admin;
import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.config.RedisCacheNames;
import com.yoyuzh.files.core.FileEntityRepository;
import com.yoyuzh.files.core.FileEntityType;
import com.yoyuzh.files.core.StoredFileEntityRepository;
import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyRepository;
import com.yoyuzh.files.policy.StoragePolicyService;
import com.yoyuzh.files.tasks.BackgroundTask;
import com.yoyuzh.files.tasks.BackgroundTaskCommandService;
import com.yoyuzh.files.tasks.BackgroundTaskType;
import lombok.RequiredArgsConstructor;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import java.util.LinkedHashMap;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class AdminStorageGovernanceService {
private final StoragePolicyRepository storagePolicyRepository;
private final StoragePolicyService storagePolicyService;
private final FileEntityRepository fileEntityRepository;
private final StoredFileEntityRepository storedFileEntityRepository;
private final BackgroundTaskCommandService backgroundTaskCommandService;
private final AdminAuditService adminAuditService;
@Transactional
@CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true)
public AdminStoragePolicyResponse createStoragePolicy(AdminStoragePolicyUpsertRequest request) {
StoragePolicy policy = new StoragePolicy();
policy.setDefaultPolicy(false);
applyStoragePolicyUpsert(policy, request);
AdminStoragePolicyResponse response = AdminStoragePolicyResponses.from(storagePolicyService, storagePolicyRepository.save(policy));
adminAuditService.record(
AdminAuditAction.CREATE_STORAGE_POLICY,
"STORAGE_POLICY",
response.id(),
"Created storage policy",
Map.of(
"name", response.name(),
"type", response.type().name(),
"enabled", response.enabled()
)
);
return response;
}
@Transactional
@CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true)
public AdminStoragePolicyResponse updateStoragePolicy(Long policyId, AdminStoragePolicyUpsertRequest request) {
StoragePolicy policy = getRequiredStoragePolicy(policyId);
applyStoragePolicyUpsert(policy, request);
AdminStoragePolicyResponse response = AdminStoragePolicyResponses.from(storagePolicyService, storagePolicyRepository.save(policy));
adminAuditService.record(
AdminAuditAction.UPDATE_STORAGE_POLICY,
"STORAGE_POLICY",
policyId,
"Updated storage policy",
Map.of(
"name", response.name(),
"type", response.type().name(),
"enabled", response.enabled()
)
);
return response;
}
@Transactional
@CacheEvict(cacheNames = RedisCacheNames.STORAGE_POLICIES, allEntries = true)
public AdminStoragePolicyResponse updateStoragePolicyStatus(Long policyId, boolean enabled) {
StoragePolicy policy = getRequiredStoragePolicy(policyId);
if (policy.isDefaultPolicy() && !enabled) {
throw new BusinessException(ErrorCode.UNKNOWN, "姒涙顓荤€涙ê鍋嶇粵鏍殣娑撳秷鍏橀崑婊呮暏");
}
policy.setEnabled(enabled);
AdminStoragePolicyResponse response = AdminStoragePolicyResponses.from(storagePolicyService, storagePolicyRepository.save(policy));
adminAuditService.record(
AdminAuditAction.UPDATE_STORAGE_POLICY_STATUS,
"STORAGE_POLICY",
policyId,
enabled ? "Enabled storage policy" : "Disabled storage policy",
Map.of("enabled", enabled)
);
return response;
}
@Transactional
public BackgroundTask createStoragePolicyMigrationTask(User user, AdminStoragePolicyMigrationCreateRequest request) {
StoragePolicy sourcePolicy = getRequiredStoragePolicy(request.sourcePolicyId());
StoragePolicy targetPolicy = getRequiredStoragePolicy(request.targetPolicyId());
if (sourcePolicy.getId().equals(targetPolicy.getId())) {
throw new BusinessException(ErrorCode.UNKNOWN, "濠ф劕鐡ㄩ崒銊х摜閻c儱鎷伴惄顔界垼鐎涙ê鍋嶇粵鏍殣娑撳秷鍏橀惄绋挎倱");
}
if (!targetPolicy.isEnabled()) {
throw new BusinessException(ErrorCode.UNKNOWN, "target storage policy must be enabled");
}
long candidateEntityCount = fileEntityRepository.countByStoragePolicyIdAndEntityType(
sourcePolicy.getId(),
FileEntityType.VERSION
);
long candidateStoredFileCount = storedFileEntityRepository.countDistinctStoredFilesByStoragePolicyIdAndEntityType(
sourcePolicy.getId(),
FileEntityType.VERSION
);
Map<String, Object> state = new LinkedHashMap<>();
state.put("sourcePolicyId", sourcePolicy.getId());
state.put("sourcePolicyName", sourcePolicy.getName());
state.put("targetPolicyId", targetPolicy.getId());
state.put("targetPolicyName", targetPolicy.getName());
state.put("candidateEntityCount", candidateEntityCount);
state.put("candidateStoredFileCount", candidateStoredFileCount);
state.put("migrationPerformed", false);
state.put("migrationMode", "skeleton");
state.put("entityType", FileEntityType.VERSION.name());
state.put("message", "storage policy migration skeleton queued; worker will validate and recount candidates without moving object data");
Map<String, Object> privateState = new LinkedHashMap<>(state);
privateState.put("taskType", BackgroundTaskType.STORAGE_POLICY_MIGRATION.name());
BackgroundTask task = backgroundTaskCommandService.createQueuedTask(
user,
BackgroundTaskType.STORAGE_POLICY_MIGRATION,
state,
privateState,
request.correlationId()
);
Map<String, Object> auditDetails = new LinkedHashMap<>();
auditDetails.put("sourcePolicyId", sourcePolicy.getId());
auditDetails.put("targetPolicyId", targetPolicy.getId());
auditDetails.put("correlationId", request.correlationId());
adminAuditService.record(
AdminAuditAction.CREATE_STORAGE_POLICY_MIGRATION_TASK,
"TASK",
task.getId(),
"Created storage policy migration task",
auditDetails
);
return task;
}
private void applyStoragePolicyUpsert(StoragePolicy policy, AdminStoragePolicyUpsertRequest request) {
if (policy.isDefaultPolicy() && !request.enabled()) {
throw new BusinessException(ErrorCode.UNKNOWN, "姒涙顓荤€涙ê鍋嶇粵鏍殣娑撳秷鍏橀崑婊呮暏");
}
validateStoragePolicyRequest(request);
policy.setName(request.name().trim());
policy.setType(request.type());
policy.setBucketName(normalizeNullable(request.bucketName()));
policy.setEndpoint(normalizeNullable(request.endpoint()));
policy.setRegion(normalizeNullable(request.region()));
policy.setPrivateBucket(request.privateBucket());
policy.setPrefix(normalizePrefix(request.prefix()));
policy.setCredentialMode(request.credentialMode());
policy.setMaxSizeBytes(request.maxSizeBytes());
policy.setCapabilitiesJson(storagePolicyService.writeCapabilities(request.capabilities()));
policy.setEnabled(request.enabled());
}
private StoragePolicy getRequiredStoragePolicy(Long policyId) {
return storagePolicyRepository.findById(policyId)
.orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "storage policy not found"));
}
private String normalizeNullable(String value) {
if (!StringUtils.hasText(value)) {
return null;
}
return value.trim();
}
private String normalizePrefix(String prefix) {
if (!StringUtils.hasText(prefix)) {
return "";
}
return prefix.trim();
}
private void validateStoragePolicyRequest(AdminStoragePolicyUpsertRequest request) {
if (request.type() == com.yoyuzh.files.policy.StoragePolicyType.LOCAL
&& request.credentialMode() != com.yoyuzh.files.policy.StoragePolicyCredentialMode.NONE) {
throw new BusinessException(ErrorCode.UNKNOWN, "閺堫剙婀寸€涙ê鍋嶇粵鏍殣韫囧懘銆忔担璺ㄦ暏 NONE 閸戭叀鐦夊Ο鈥崇础");
}
if (request.type() == com.yoyuzh.files.policy.StoragePolicyType.S3_COMPATIBLE
&& !StringUtils.hasText(request.bucketName())) {
throw new BusinessException(ErrorCode.UNKNOWN, "S3 鐎涙ê鍋嶇粵鏍殣韫囧懘銆忛幓鎰返 bucketName");
}
}
}

View File

@@ -0,0 +1,29 @@
package com.yoyuzh.admin;
import com.yoyuzh.config.RedisCacheNames;
import com.yoyuzh.files.policy.StoragePolicyRepository;
import com.yoyuzh.files.policy.StoragePolicyService;
import lombok.RequiredArgsConstructor;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
@RequiredArgsConstructor
public class AdminStoragePolicyQueryService {
private final StoragePolicyRepository storagePolicyRepository;
private final StoragePolicyService storagePolicyService;
@Cacheable(cacheNames = RedisCacheNames.STORAGE_POLICIES, key = "'all'")
public List<AdminStoragePolicyResponse> listStoragePolicies() {
return storagePolicyRepository.findAll(Sort.by(Sort.Direction.DESC, "defaultPolicy")
.and(Sort.by(Sort.Direction.DESC, "enabled"))
.and(Sort.by(Sort.Direction.ASC, "id")))
.stream()
.map(policy -> AdminStoragePolicyResponses.from(storagePolicyService, policy))
.toList();
}
}

View File

@@ -0,0 +1,30 @@
package com.yoyuzh.admin;
import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyService;
final class AdminStoragePolicyResponses {
private AdminStoragePolicyResponses() {
}
static AdminStoragePolicyResponse from(StoragePolicyService storagePolicyService, StoragePolicy policy) {
return new AdminStoragePolicyResponse(
policy.getId(),
policy.getName(),
policy.getType(),
policy.getBucketName(),
policy.getEndpoint(),
policy.getRegion(),
policy.isPrivateBucket(),
policy.getPrefix(),
policy.getCredentialMode(),
policy.getMaxSizeBytes(),
storagePolicyService.readCapabilities(policy),
policy.isEnabled(),
policy.isDefaultPolicy(),
policy.getCreatedAt(),
policy.getUpdatedAt()
);
}
}

View File

@@ -0,0 +1,148 @@
package com.yoyuzh.admin;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.tasks.BackgroundTask;
import com.yoyuzh.files.tasks.BackgroundTaskFailureCategory;
import com.yoyuzh.files.tasks.BackgroundTaskRepository;
import com.yoyuzh.files.tasks.BackgroundTaskStatus;
import com.yoyuzh.files.tasks.BackgroundTaskType;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class AdminTaskQueryService {
private final BackgroundTaskRepository backgroundTaskRepository;
private final UserRepository userRepository;
private final ObjectMapper objectMapper;
public PageResponse<AdminTaskResponse> listTasks(int page,
int size,
String userQuery,
BackgroundTaskType type,
BackgroundTaskStatus status,
BackgroundTaskFailureCategory failureCategory,
AdminTaskLeaseState leaseState) {
String failureCategoryPattern = failureCategory == null
? null
: "\"failureCategory\":\"" + failureCategory.name() + "\"";
Page<BackgroundTask> result = backgroundTaskRepository.searchAdminTasks(
normalizeQuery(userQuery),
type,
status,
failureCategoryPattern,
leaseState == null ? null : leaseState.name(),
LocalDateTime.now(),
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
Map<Long, User> ownerById = userRepository.findAllById(result.getContent().stream()
.map(BackgroundTask::getUserId)
.collect(Collectors.toSet()))
.stream()
.collect(Collectors.toMap(User::getId, user -> user));
return new PageResponse<>(
result.getContent().stream()
.map(task -> toAdminTaskResponse(task, ownerById.get(task.getUserId())))
.toList(),
result.getTotalElements(),
page,
size
);
}
public AdminTaskResponse getTask(Long taskId) {
BackgroundTask task = backgroundTaskRepository.findById(taskId)
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "task not found"));
User owner = userRepository.findById(task.getUserId()).orElse(null);
return toAdminTaskResponse(task, owner);
}
private AdminTaskResponse toAdminTaskResponse(BackgroundTask task, User owner) {
Map<String, Object> state = parseState(task.getPublicStateJson());
return new AdminTaskResponse(
task.getId(),
task.getType(),
task.getStatus(),
task.getUserId(),
owner == null ? null : owner.getUsername(),
owner == null ? null : owner.getEmail(),
task.getPublicStateJson(),
task.getCorrelationId(),
task.getErrorMessage(),
task.getAttemptCount(),
task.getMaxAttempts(),
task.getNextRunAt(),
task.getLeaseOwner(),
task.getLeaseExpiresAt(),
task.getHeartbeatAt(),
task.getCreatedAt(),
task.getUpdatedAt(),
task.getFinishedAt(),
readStringState(state, "failureCategory"),
readBooleanState(state, "retryScheduled"),
readStringState(state, "workerOwner"),
resolveLeaseState(task)
);
}
private Map<String, Object> parseState(String json) {
if (!StringUtils.hasText(json)) {
return Map.of();
}
try {
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
return Map.of();
}
}
private String readStringState(Map<String, Object> state, String key) {
Object value = state.get(key);
return value == null ? null : String.valueOf(value);
}
private Boolean readBooleanState(Map<String, Object> state, String key) {
Object value = state.get(key);
if (value instanceof Boolean boolValue) {
return boolValue;
}
if (value instanceof String stringValue) {
return Boolean.parseBoolean(stringValue);
}
return null;
}
private AdminTaskLeaseState resolveLeaseState(BackgroundTask task) {
if (!StringUtils.hasText(task.getLeaseOwner()) || task.getLeaseExpiresAt() == null) {
return AdminTaskLeaseState.NONE;
}
return task.getLeaseExpiresAt().isBefore(LocalDateTime.now())
? AdminTaskLeaseState.EXPIRED
: AdminTaskLeaseState.ACTIVE;
}
private String normalizeQuery(String query) {
if (query == null) {
return "";
}
return query.trim();
}
}

View File

@@ -0,0 +1,227 @@
package com.yoyuzh.admin;
import com.yoyuzh.auth.AuthSessionPolicy;
import com.yoyuzh.auth.AuthTokenInvalidationService;
import com.yoyuzh.auth.PasswordPolicy;
import com.yoyuzh.auth.RefreshTokenService;
import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.auth.UserRole;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.core.StoredFileRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.security.SecureRandom;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class AdminUserGovernanceService {
private final UserRepository userRepository;
private final StoredFileRepository storedFileRepository;
private final PasswordEncoder passwordEncoder;
private final RefreshTokenService refreshTokenService;
private final AuthTokenInvalidationService authTokenInvalidationService;
private final AuthSessionPolicy authSessionPolicy;
private final AdminAuditService adminAuditService;
private final SecureRandom secureRandom = new SecureRandom();
public PageResponse<AdminUserResponse> listUsers(int page, int size, String query) {
Page<User> result = userRepository.searchByUsernameOrEmail(
normalizeQuery(query),
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
List<User> users = result.getContent();
Map<Long, Long> usedStorageByUserId = loadUsedStorageByUserIds(users);
return new PageResponse<>(
users.stream()
.map(user -> toUserResponse(user, usedStorageByUserId.getOrDefault(user.getId(), 0L)))
.toList(),
result.getTotalElements(),
page,
size
);
}
@Transactional
public AdminUserResponse updateUserRole(Long userId, UserRole role) {
User user = getRequiredUser(userId);
user.setRole(role);
AdminUserResponse response = toUserResponse(userRepository.save(user));
adminAuditService.record(
AdminAuditAction.UPDATE_USER_ROLE,
"USER",
userId,
"Updated user role",
Map.of("role", role.name())
);
return response;
}
@Transactional
public AdminUserResponse updateUserBanned(Long userId, boolean banned) {
User user = getRequiredUser(userId);
user.setBanned(banned);
authTokenInvalidationService.revokeAccessTokensForUser(user.getId());
authSessionPolicy.rotateAllActiveSessions(user);
refreshTokenService.revokeAllForUser(user.getId());
AdminUserResponse response = toUserResponse(userRepository.save(user));
adminAuditService.record(
AdminAuditAction.UPDATE_USER_BANNED,
"USER",
userId,
banned ? "Banned user" : "Unbanned user",
Map.of("banned", banned)
);
return response;
}
@Transactional
public AdminUserResponse updateUserPassword(Long userId, String newPassword) {
return updateUserPasswordInternal(userId, newPassword, AdminAuditAction.UPDATE_USER_PASSWORD);
}
@Transactional
public AdminUserResponse updateUserStorageQuota(Long userId, long storageQuotaBytes) {
User user = getRequiredUser(userId);
user.setStorageQuotaBytes(storageQuotaBytes);
AdminUserResponse response = toUserResponse(userRepository.save(user));
adminAuditService.record(
AdminAuditAction.UPDATE_USER_STORAGE_QUOTA,
"USER",
userId,
"Updated user storage quota",
Map.of("storageQuotaBytes", storageQuotaBytes)
);
return response;
}
@Transactional
public AdminUserResponse updateUserMaxUploadSize(Long userId, long maxUploadSizeBytes) {
User user = getRequiredUser(userId);
user.setMaxUploadSizeBytes(maxUploadSizeBytes);
AdminUserResponse response = toUserResponse(userRepository.save(user));
adminAuditService.record(
AdminAuditAction.UPDATE_USER_MAX_UPLOAD_SIZE,
"USER",
userId,
"Updated user max upload size",
Map.of("maxUploadSizeBytes", maxUploadSizeBytes)
);
return response;
}
@Transactional
public AdminPasswordResetResponse resetUserPassword(Long userId) {
String temporaryPassword = generateTemporaryPassword();
updateUserPasswordInternal(userId, temporaryPassword, AdminAuditAction.RESET_USER_PASSWORD);
return new AdminPasswordResetResponse(temporaryPassword);
}
private AdminUserResponse updateUserPasswordInternal(Long userId, String newPassword, AdminAuditAction action) {
if (!PasswordPolicy.isStrong(newPassword)) {
throw new BusinessException(ErrorCode.UNKNOWN, PasswordPolicy.VALIDATION_MESSAGE);
}
User user = getRequiredUser(userId);
user.setPasswordHash(passwordEncoder.encode(newPassword));
authTokenInvalidationService.revokeAccessTokensForUser(user.getId());
authSessionPolicy.rotateAllActiveSessions(user);
refreshTokenService.revokeAllForUser(user.getId());
AdminUserResponse response = toUserResponse(userRepository.save(user));
Map<String, Object> details = new LinkedHashMap<>();
details.put("passwordLength", newPassword.length());
details.put("temporaryPassword", action == AdminAuditAction.RESET_USER_PASSWORD);
adminAuditService.record(
action,
"USER",
userId,
action == AdminAuditAction.RESET_USER_PASSWORD
? "Reset user password"
: "Updated user password",
details
);
return response;
}
private AdminUserResponse toUserResponse(User user) {
return toUserResponse(user, storedFileRepository.sumFileSizeByUserId(user.getId()));
}
private AdminUserResponse toUserResponse(User user, long usedStorageBytes) {
return new AdminUserResponse(
user.getId(),
user.getUsername(),
user.getEmail(),
user.getPhoneNumber(),
user.getCreatedAt(),
user.getRole(),
user.isBanned(),
usedStorageBytes,
user.getStorageQuotaBytes(),
user.getMaxUploadSizeBytes()
);
}
private Map<Long, Long> loadUsedStorageByUserIds(List<User> users) {
Set<Long> userIds = users.stream()
.map(User::getId)
.filter(id -> id != null)
.collect(Collectors.toSet());
if (userIds.isEmpty()) {
return Map.of();
}
return storedFileRepository.sumFileSizeByUserIds(userIds).stream()
.collect(Collectors.toMap(
StoredFileRepository.UserStorageUsageProjection::getUserId,
projection -> projection.getUsedStorageBytes() == null ? 0L : projection.getUsedStorageBytes()
));
}
private User getRequiredUser(Long userId) {
return userRepository.findById(userId)
.orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "user not found"));
}
private String normalizeQuery(String query) {
if (query == null) {
return "";
}
return query.trim();
}
private String generateTemporaryPassword() {
String lowers = "abcdefghjkmnpqrstuvwxyz";
String uppers = "ABCDEFGHJKMNPQRSTUVWXYZ";
String digits = "23456789";
String specials = "!@#$%^&*";
String all = lowers + uppers + digits + specials;
char[] password = new char[12];
password[0] = lowers.charAt(secureRandom.nextInt(lowers.length()));
password[1] = uppers.charAt(secureRandom.nextInt(uppers.length()));
password[2] = digits.charAt(secureRandom.nextInt(digits.length()));
password[3] = specials.charAt(secureRandom.nextInt(specials.length()));
for (int i = 4; i < password.length; i += 1) {
password[i] = all.charAt(secureRandom.nextInt(all.length()));
}
for (int i = password.length - 1; i > 0; i -= 1) {
int j = secureRandom.nextInt(i + 1);
char tmp = password[i];
password[i] = password[j];
password[j] = tmp;
}
return new String(password);
}
}

View File

@@ -0,0 +1,13 @@
package com.yoyuzh.api.v2.files;
import java.time.LocalDateTime;
public record UploadSessionRuntimeStateV2Response(
String phase,
long uploadedBytes,
int uploadedPartCount,
Integer progressPercent,
LocalDateTime lastUpdatedAt,
LocalDateTime expiresAt
) {
}

View File

@@ -4,6 +4,7 @@ import com.yoyuzh.api.v2.ApiV2Response;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.files.upload.UploadSession; import com.yoyuzh.files.upload.UploadSession;
import com.yoyuzh.files.upload.UploadSessionRuntimeState;
import com.yoyuzh.files.upload.UploadSessionCreateCommand; import com.yoyuzh.files.upload.UploadSessionCreateCommand;
import com.yoyuzh.files.upload.UploadSessionUploadMode; import com.yoyuzh.files.upload.UploadSessionUploadMode;
import com.yoyuzh.files.upload.UploadSessionPartCommand; import com.yoyuzh.files.upload.UploadSessionPartCommand;
@@ -142,10 +143,24 @@ public class UploadSessionV2Controller {
session.getExpiresAt(), session.getExpiresAt(),
session.getCreatedAt(), session.getCreatedAt(),
session.getUpdatedAt(), session.getUpdatedAt(),
uploadSessionService.getRuntimeState(session.getSessionId())
.map(this::toRuntimeResponse)
.orElse(null),
toStrategyResponse(session.getSessionId(), uploadMode) toStrategyResponse(session.getSessionId(), uploadMode)
); );
} }
private UploadSessionRuntimeStateV2Response toRuntimeResponse(UploadSessionRuntimeState runtimeState) {
return new UploadSessionRuntimeStateV2Response(
runtimeState.phase(),
runtimeState.uploadedBytes(),
runtimeState.uploadedPartCount(),
runtimeState.progressPercent(),
runtimeState.lastUpdatedAt(),
runtimeState.expiresAt()
);
}
private UploadSessionV2StrategyResponse toStrategyResponse(String sessionId, UploadSessionUploadMode uploadMode) { private UploadSessionV2StrategyResponse toStrategyResponse(String sessionId, UploadSessionUploadMode uploadMode) {
String sessionBasePath = "/api/v2/files/upload-sessions/" + sessionId; String sessionBasePath = "/api/v2/files/upload-sessions/" + sessionId;
return switch (uploadMode) { return switch (uploadMode) {

View File

@@ -19,6 +19,7 @@ public record UploadSessionV2Response(
LocalDateTime expiresAt, LocalDateTime expiresAt,
LocalDateTime createdAt, LocalDateTime createdAt,
LocalDateTime updatedAt, LocalDateTime updatedAt,
UploadSessionRuntimeStateV2Response runtime,
UploadSessionV2StrategyResponse strategy UploadSessionV2StrategyResponse strategy
) { ) {
} }

View File

@@ -5,7 +5,7 @@ import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.tasks.BackgroundTask; import com.yoyuzh.files.tasks.BackgroundTask;
import com.yoyuzh.files.tasks.BackgroundTaskService; import com.yoyuzh.files.tasks.BackgroundTaskCommandService;
import com.yoyuzh.files.tasks.BackgroundTaskType; import com.yoyuzh.files.tasks.BackgroundTaskType;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Max;
@@ -29,7 +29,7 @@ import org.springframework.web.bind.annotation.RestController;
@RequiredArgsConstructor @RequiredArgsConstructor
public class BackgroundTaskV2Controller { public class BackgroundTaskV2Controller {
private final BackgroundTaskService backgroundTaskService; private final BackgroundTaskCommandService backgroundTaskCommandService;
private final CustomUserDetailsService userDetailsService; private final CustomUserDetailsService userDetailsService;
@GetMapping @GetMapping
@@ -37,7 +37,7 @@ public class BackgroundTaskV2Controller {
@RequestParam(defaultValue = "0") @Min(0) int page, @RequestParam(defaultValue = "0") @Min(0) int page,
@RequestParam(defaultValue = "20") @Min(1) @Max(100) int size) { @RequestParam(defaultValue = "20") @Min(1) @Max(100) int size) {
User user = userDetailsService.loadDomainUser(userDetails.getUsername()); User user = userDetailsService.loadDomainUser(userDetails.getUsername());
var result = backgroundTaskService.listOwnedTasks( var result = backgroundTaskCommandService.listOwnedTasks(
user, user,
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt")) PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
); );
@@ -53,21 +53,21 @@ public class BackgroundTaskV2Controller {
public ApiV2Response<BackgroundTaskResponse> get(@AuthenticationPrincipal UserDetails userDetails, public ApiV2Response<BackgroundTaskResponse> get(@AuthenticationPrincipal UserDetails userDetails,
@PathVariable Long id) { @PathVariable Long id) {
User user = userDetailsService.loadDomainUser(userDetails.getUsername()); User user = userDetailsService.loadDomainUser(userDetails.getUsername());
return ApiV2Response.success(toResponse(backgroundTaskService.getOwnedTask(user, id))); return ApiV2Response.success(toResponse(backgroundTaskCommandService.getOwnedTask(user, id)));
} }
@DeleteMapping("/{id}") @DeleteMapping("/{id}")
public ApiV2Response<BackgroundTaskResponse> cancel(@AuthenticationPrincipal UserDetails userDetails, public ApiV2Response<BackgroundTaskResponse> cancel(@AuthenticationPrincipal UserDetails userDetails,
@PathVariable Long id) { @PathVariable Long id) {
User user = userDetailsService.loadDomainUser(userDetails.getUsername()); User user = userDetailsService.loadDomainUser(userDetails.getUsername());
return ApiV2Response.success(toResponse(backgroundTaskService.cancelOwnedTask(user, id))); return ApiV2Response.success(toResponse(backgroundTaskCommandService.cancelOwnedTask(user, id)));
} }
@PostMapping("/{id}/retry") @PostMapping("/{id}/retry")
public ApiV2Response<BackgroundTaskResponse> retry(@AuthenticationPrincipal UserDetails userDetails, public ApiV2Response<BackgroundTaskResponse> retry(@AuthenticationPrincipal UserDetails userDetails,
@PathVariable Long id) { @PathVariable Long id) {
User user = userDetailsService.loadDomainUser(userDetails.getUsername()); User user = userDetailsService.loadDomainUser(userDetails.getUsername());
return ApiV2Response.success(toResponse(backgroundTaskService.retryOwnedTask(user, id))); return ApiV2Response.success(toResponse(backgroundTaskCommandService.retryOwnedTask(user, id)));
} }
@PostMapping("/archive") @PostMapping("/archive")
@@ -92,7 +92,7 @@ public class BackgroundTaskV2Controller {
BackgroundTaskType type, BackgroundTaskType type,
CreateBackgroundTaskRequest request) { CreateBackgroundTaskRequest request) {
User user = userDetailsService.loadDomainUser(userDetails.getUsername()); User user = userDetailsService.loadDomainUser(userDetails.getUsername());
BackgroundTask task = backgroundTaskService.createQueuedFileTask( BackgroundTask task = backgroundTaskCommandService.createQueuedFileTask(
user, user,
type, type,
request.fileId(), request.fileId(),

View File

@@ -44,6 +44,8 @@ public class AuthService {
private final AuthenticationManager authenticationManager; private final AuthenticationManager authenticationManager;
private final JwtTokenProvider jwtTokenProvider; private final JwtTokenProvider jwtTokenProvider;
private final RefreshTokenService refreshTokenService; private final RefreshTokenService refreshTokenService;
private final AuthTokenInvalidationService authTokenInvalidationService;
private final AuthSessionPolicy authSessionPolicy;
private final FileService fileService; private final FileService fileService;
private final FileContentStorage fileContentStorage; private final FileContentStorage fileContentStorage;
private final RegistrationInviteService registrationInviteService; private final RegistrationInviteService registrationInviteService;
@@ -115,13 +117,20 @@ public class AuthService {
} }
final String finalCandidate = candidate; final String finalCandidate = candidate;
User user = userRepository.findByUsername(finalCandidate).orElseGet(() -> { UserRole desiredRole = resolveDevLoginRole(finalCandidate);
User user = userRepository.findByUsername(finalCandidate).map(existing -> {
if (existing.getRole() != desiredRole) {
existing.setRole(desiredRole);
return userRepository.save(existing);
}
return existing;
}).orElseGet(() -> {
User created = new User(); User created = new User();
created.setUsername(finalCandidate); created.setUsername(finalCandidate);
created.setDisplayName(finalCandidate); created.setDisplayName(finalCandidate);
created.setEmail(finalCandidate + "@dev.local"); created.setEmail(finalCandidate + "@dev.local");
created.setPasswordHash(passwordEncoder.encode("1")); created.setPasswordHash(passwordEncoder.encode("1"));
created.setRole(UserRole.USER); created.setRole(desiredRole);
created.setPreferredLanguage("zh-CN"); created.setPreferredLanguage("zh-CN");
return userRepository.save(created); return userRepository.save(created);
}); });
@@ -291,6 +300,7 @@ public class AuthService {
} }
private AuthResponse issueFreshTokens(User user, AuthClientType clientType) { private AuthResponse issueFreshTokens(User user, AuthClientType clientType) {
authTokenInvalidationService.revokeAccessTokensForUser(user.getId(), clientType);
refreshTokenService.revokeAllForUser(user.getId(), clientType); refreshTokenService.revokeAllForUser(user.getId(), clientType);
return issueTokens(user, refreshTokenService.issueRefreshToken(user, clientType), clientType); return issueTokens(user, refreshTokenService.issueRefreshToken(user, clientType), clientType);
} }
@@ -300,31 +310,20 @@ public class AuthService {
String accessToken = jwtTokenProvider.generateAccessToken( String accessToken = jwtTokenProvider.generateAccessToken(
sessionUser.getId(), sessionUser.getId(),
sessionUser.getUsername(), sessionUser.getUsername(),
getActiveSessionId(sessionUser, clientType), authSessionPolicy.getActiveSessionId(sessionUser, clientType),
clientType clientType
); );
return AuthResponse.issued(accessToken, refreshToken, toProfile(sessionUser)); return AuthResponse.issued(accessToken, refreshToken, toProfile(sessionUser));
} }
private User rotateActiveSession(User user, AuthClientType clientType) { private User rotateActiveSession(User user, AuthClientType clientType) {
String nextSessionId = UUID.randomUUID().toString(); authSessionPolicy.rotateActiveSession(user, clientType);
if (clientType == AuthClientType.MOBILE) {
user.setMobileActiveSessionId(nextSessionId);
} else {
user.setDesktopActiveSessionId(nextSessionId);
user.setActiveSessionId(nextSessionId);
}
return userRepository.save(user); return userRepository.save(user);
} }
private void rotateAllActiveSessions(User user) { private void rotateAllActiveSessions(User user) {
user.setActiveSessionId(UUID.randomUUID().toString()); authTokenInvalidationService.revokeAccessTokensForUser(user.getId());
user.setDesktopActiveSessionId(UUID.randomUUID().toString()); authSessionPolicy.rotateAllActiveSessions(user);
user.setMobileActiveSessionId(UUID.randomUUID().toString());
}
private String getActiveSessionId(User user, AuthClientType clientType) {
return clientType == AuthClientType.MOBILE ? user.getMobileActiveSessionId() : user.getDesktopActiveSessionId();
} }
private String normalizeOptionalText(String value) { private String normalizeOptionalText(String value) {
@@ -335,6 +334,16 @@ public class AuthService {
return trimmed.isEmpty() ? null : trimmed; return trimmed.isEmpty() ? null : trimmed;
} }
private UserRole resolveDevLoginRole(String username) {
if ("admin".equalsIgnoreCase(username)) {
return UserRole.ADMIN;
}
if ("operator".equalsIgnoreCase(username) || "moderator".equalsIgnoreCase(username)) {
return UserRole.MODERATOR;
}
return UserRole.USER;
}
private String normalizePreferredLanguage(String preferredLanguage) { private String normalizePreferredLanguage(String preferredLanguage) {
if (preferredLanguage == null || preferredLanguage.trim().isEmpty()) { if (preferredLanguage == null || preferredLanguage.trim().isEmpty()) {
return "zh-CN"; return "zh-CN";

View File

@@ -0,0 +1,35 @@
package com.yoyuzh.auth;
import org.springframework.stereotype.Component;
import java.util.UUID;
@Component
public class AuthSessionPolicy {
public void rotateActiveSession(User user, AuthClientType clientType) {
String nextSessionId = nextSessionId();
if (clientType == AuthClientType.MOBILE) {
user.setMobileActiveSessionId(nextSessionId);
return;
}
user.setDesktopActiveSessionId(nextSessionId);
user.setActiveSessionId(nextSessionId);
}
public void rotateAllActiveSessions(User user) {
user.setActiveSessionId(nextSessionId());
user.setDesktopActiveSessionId(nextSessionId());
user.setMobileActiveSessionId(nextSessionId());
}
public String getActiveSessionId(User user, AuthClientType clientType) {
return clientType == AuthClientType.MOBILE
? user.getMobileActiveSessionId()
: user.getDesktopActiveSessionId();
}
private String nextSessionId() {
return UUID.randomUUID().toString();
}
}

View File

@@ -0,0 +1,108 @@
package com.yoyuzh.auth;
import com.yoyuzh.config.AppRedisProperties;
import com.yoyuzh.config.JwtProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.time.Duration;
import java.time.Instant;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public class AuthTokenInvalidationService {
private final StringRedisTemplate redisTemplate;
private final AppRedisProperties redisProperties;
private final JwtProperties jwtProperties;
public AuthTokenInvalidationService(StringRedisTemplate redisTemplate,
AppRedisProperties redisProperties,
JwtProperties jwtProperties) {
this.redisTemplate = redisTemplate;
this.redisProperties = redisProperties;
this.jwtProperties = jwtProperties;
}
public void revokeAccessTokensForUser(Long userId) {
revokeAccessTokensForUser(userId, AuthClientType.DESKTOP);
revokeAccessTokensForUser(userId, AuthClientType.MOBILE);
}
public void revokeAccessTokensForUser(Long userId, AuthClientType clientType) {
if (userId == null || clientType == null) {
return;
}
redisTemplate.opsForValue().set(
buildAccessInvalidationKey(userId, clientType),
Long.toString(Instant.now().getEpochSecond()),
Duration.ofSeconds(jwtProperties.getAccessExpirationSeconds() + redisProperties.getTtlBufferSeconds())
);
}
public boolean isAccessTokenRevoked(Long userId, AuthClientType clientType, Instant issuedAt) {
if (userId == null || clientType == null || issuedAt == null) {
return false;
}
String rawValue = redisTemplate.opsForValue().get(buildAccessInvalidationKey(userId, clientType));
if (!StringUtils.hasText(rawValue)) {
return false;
}
long revokedBeforeEpochSecond = normalizeRevokedBefore(rawValue);
if (revokedBeforeEpochSecond <= 0L) {
return false;
}
return issuedAt.getEpochSecond() < revokedBeforeEpochSecond;
}
public void blacklistRefreshTokenHash(String tokenHash, Instant expiresAt) {
if (!StringUtils.hasText(tokenHash) || expiresAt == null) {
return;
}
Duration ttl = Duration.between(Instant.now(), expiresAt)
.plusSeconds(redisProperties.getTtlBufferSeconds());
if (ttl.isNegative() || ttl.isZero()) {
ttl = Duration.ofSeconds(redisProperties.getTtlBufferSeconds());
}
redisTemplate.opsForValue().set(buildRefreshTokenBlacklistKey(tokenHash), "1", ttl);
}
public boolean isRefreshTokenHashBlacklisted(String tokenHash) {
if (!StringUtils.hasText(tokenHash)) {
return false;
}
return Boolean.TRUE.equals(redisTemplate.hasKey(buildRefreshTokenBlacklistKey(tokenHash)));
}
private String buildAccessInvalidationKey(Long userId, AuthClientType clientType) {
return buildAuthKey("access-revoked-before", userId.toString(), clientType.name());
}
private String buildRefreshTokenBlacklistKey(String tokenHash) {
return buildAuthKey("refresh-blacklist", tokenHash);
}
private long normalizeRevokedBefore(String rawValue) {
long parsed = Long.parseLong(rawValue);
if (parsed > 9_999_999_999L) {
return parsed / 1000L;
}
return parsed;
}
private String buildAuthKey(String... segments) {
StringBuilder builder = new StringBuilder();
builder.append(redisProperties.getKeyPrefix())
.append(':')
.append(redisProperties.getNamespaces().getAuth());
for (String segment : segments) {
if (!StringUtils.hasText(segment)) {
continue;
}
builder.append(':').append(segment.trim());
}
return builder.toString();
}
}

View File

@@ -79,6 +79,11 @@ public class JwtTokenProvider {
return uid == null ? null : Long.parseLong(uid.toString()); return uid == null ? null : Long.parseLong(uid.toString());
} }
public Instant getIssuedAt(String token) {
Date issuedAt = parseClaims(token).getIssuedAt();
return issuedAt == null ? null : issuedAt.toInstant();
}
public String getSessionId(String token) { public String getSessionId(String token) {
Object sessionId = parseClaims(token).get("sid"); Object sessionId = parseClaims(token).get("sid");
return sessionId == null ? null : sessionId.toString(); return sessionId == null ? null : sessionId.toString();

View File

@@ -0,0 +1,37 @@
package com.yoyuzh.auth;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.time.Instant;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "false", matchIfMissing = true)
public class NoOpAuthTokenInvalidationService extends AuthTokenInvalidationService {
public NoOpAuthTokenInvalidationService() {
super(null, null, null);
}
@Override
public void revokeAccessTokensForUser(Long userId) {
}
@Override
public void revokeAccessTokensForUser(Long userId, AuthClientType clientType) {
}
@Override
public boolean isAccessTokenRevoked(Long userId, AuthClientType clientType, Instant issuedAt) {
return false;
}
@Override
public void blacklistRefreshTokenHash(String tokenHash, Instant expiresAt) {
}
@Override
public boolean isRefreshTokenHashBlacklisted(String tokenHash) {
return false;
}
}

View File

@@ -8,6 +8,7 @@ import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param; import org.springframework.data.repository.query.Param;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional; import java.util.Optional;
public interface RefreshTokenRepository extends JpaRepository<RefreshToken, Long> { public interface RefreshTokenRepository extends JpaRepository<RefreshToken, Long> {
@@ -34,4 +35,19 @@ public interface RefreshTokenRepository extends JpaRepository<RefreshToken, Long
int revokeAllActiveByUserIdAndClientType(@Param("userId") Long userId, int revokeAllActiveByUserIdAndClientType(@Param("userId") Long userId,
@Param("clientType") String clientType, @Param("clientType") String clientType,
@Param("revokedAt") LocalDateTime revokedAt); @Param("revokedAt") LocalDateTime revokedAt);
@Query("""
select token from RefreshToken token
where token.user.id = :userId and token.revoked = false and token.expiresAt > :now
""")
List<RefreshToken> findActiveByUserId(@Param("userId") Long userId, @Param("now") LocalDateTime now);
@Query("""
select token from RefreshToken token
where token.user.id = :userId and token.revoked = false and token.expiresAt > :now
and (token.clientType = :clientType or (:clientType = 'DESKTOP' and token.clientType is null))
""")
List<RefreshToken> findActiveByUserIdAndClientType(@Param("userId") Long userId,
@Param("clientType") String clientType,
@Param("now") LocalDateTime now);
} }

View File

@@ -11,9 +11,12 @@ import java.nio.charset.StandardCharsets;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom; import java.security.SecureRandom;
import java.time.Instant;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Base64; import java.util.Base64;
import java.util.HexFormat; import java.util.HexFormat;
import java.util.List;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -23,6 +26,7 @@ public class RefreshTokenService {
private final RefreshTokenRepository refreshTokenRepository; private final RefreshTokenRepository refreshTokenRepository;
private final JwtProperties jwtProperties; private final JwtProperties jwtProperties;
private final AuthTokenInvalidationService authTokenInvalidationService;
private final SecureRandom secureRandom = new SecureRandom(); private final SecureRandom secureRandom = new SecureRandom();
@Transactional @Transactional
@@ -47,7 +51,12 @@ public class RefreshTokenService {
@Transactional(noRollbackFor = BusinessException.class) @Transactional(noRollbackFor = BusinessException.class)
public RotatedRefreshToken rotateRefreshToken(String rawToken) { public RotatedRefreshToken rotateRefreshToken(String rawToken) {
RefreshToken existing = refreshTokenRepository.findForUpdateByTokenHash(hashToken(rawToken)) String tokenHash = hashToken(rawToken);
if (authTokenInvalidationService.isRefreshTokenHashBlacklisted(tokenHash)) {
throw new BusinessException(ErrorCode.NOT_LOGGED_IN, "刷新令牌无效或已使用");
}
RefreshToken existing = refreshTokenRepository.findForUpdateByTokenHash(tokenHash)
.orElseThrow(() -> new BusinessException(ErrorCode.NOT_LOGGED_IN, "刷新令牌无效")); .orElseThrow(() -> new BusinessException(ErrorCode.NOT_LOGGED_IN, "刷新令牌无效"));
if (existing.isRevoked()) { if (existing.isRevoked()) {
@@ -56,12 +65,14 @@ public class RefreshTokenService {
if (existing.getExpiresAt().isBefore(LocalDateTime.now())) { if (existing.getExpiresAt().isBefore(LocalDateTime.now())) {
existing.revoke(LocalDateTime.now()); existing.revoke(LocalDateTime.now());
authTokenInvalidationService.blacklistRefreshTokenHash(existing.getTokenHash(), toInstant(existing.getExpiresAt()));
throw new BusinessException(ErrorCode.NOT_LOGGED_IN, "刷新令牌已过期"); throw new BusinessException(ErrorCode.NOT_LOGGED_IN, "刷新令牌已过期");
} }
User user = existing.getUser(); User user = existing.getUser();
AuthClientType clientType = AuthClientType.fromHeader(existing.getClientType()); AuthClientType clientType = AuthClientType.fromHeader(existing.getClientType());
existing.revoke(LocalDateTime.now()); existing.revoke(LocalDateTime.now());
authTokenInvalidationService.blacklistRefreshTokenHash(existing.getTokenHash(), toInstant(existing.getExpiresAt()));
revokeAllForUser(user.getId(), clientType); revokeAllForUser(user.getId(), clientType);
String nextRefreshToken = issueRefreshToken(user, clientType); String nextRefreshToken = issueRefreshToken(user, clientType);
@@ -70,12 +81,18 @@ public class RefreshTokenService {
@Transactional @Transactional
public void revokeAllForUser(Long userId) { public void revokeAllForUser(Long userId) {
refreshTokenRepository.revokeAllActiveByUserId(userId, LocalDateTime.now()); LocalDateTime now = LocalDateTime.now();
List<RefreshToken> tokens = refreshTokenRepository.findActiveByUserId(userId, now);
refreshTokenRepository.revokeAllActiveByUserId(userId, now);
blacklistRefreshTokens(tokens);
} }
@Transactional @Transactional
public void revokeAllForUser(Long userId, AuthClientType clientType) { public void revokeAllForUser(Long userId, AuthClientType clientType) {
refreshTokenRepository.revokeAllActiveByUserIdAndClientType(userId, clientType.name(), LocalDateTime.now()); LocalDateTime now = LocalDateTime.now();
List<RefreshToken> tokens = refreshTokenRepository.findActiveByUserIdAndClientType(userId, clientType.name(), now);
refreshTokenRepository.revokeAllActiveByUserIdAndClientType(userId, clientType.name(), now);
blacklistRefreshTokens(tokens);
} }
private String generateRawToken() { private String generateRawToken() {
@@ -97,6 +114,16 @@ public class RefreshTokenService {
} }
} }
private void blacklistRefreshTokens(List<RefreshToken> tokens) {
for (RefreshToken token : tokens) {
authTokenInvalidationService.blacklistRefreshTokenHash(token.getTokenHash(), toInstant(token.getExpiresAt()));
}
}
private Instant toInstant(LocalDateTime dateTime) {
return dateTime.atZone(ZoneId.systemDefault()).toInstant();
}
public record RotatedRefreshToken(User user, String refreshToken, AuthClientType clientType) { public record RotatedRefreshToken(User user, String refreshToken, AuthClientType clientType) {
} }
} }

View File

@@ -28,6 +28,20 @@ public class RegistrationInviteService {
return ensureCurrentState().getInviteCode(); return ensureCurrentState().getInviteCode();
} }
@Transactional
public String updateCurrentInviteCode(String inviteCode) {
RegistrationInviteState state = ensureCurrentStateForUpdate();
state.setInviteCode(requireValidInviteCode(inviteCode));
return registrationInviteStateRepository.save(state).getInviteCode();
}
@Transactional
public String rotateCurrentInviteCode() {
RegistrationInviteState state = ensureCurrentStateForUpdate();
state.setInviteCode(generateNextInviteCode(state.getInviteCode()));
return registrationInviteStateRepository.save(state).getInviteCode();
}
@Transactional @Transactional
public void consumeInviteCode(String inviteCode) { public void consumeInviteCode(String inviteCode) {
RegistrationInviteState state = ensureCurrentStateForUpdate(); RegistrationInviteState state = ensureCurrentStateForUpdate();
@@ -93,4 +107,15 @@ public class RegistrationInviteService {
private String normalize(String value) { private String normalize(String value) {
return value == null ? "" : value.trim(); return value == null ? "" : value.trim();
} }
private String requireValidInviteCode(String inviteCode) {
String normalized = normalize(inviteCode);
if (!StringUtils.hasText(normalized)) {
throw new BusinessException(ErrorCode.UNKNOWN, "邀请码不能为空");
}
if (normalized.length() > 64) {
throw new BusinessException(ErrorCode.UNKNOWN, "邀请码长度不能超过 64 个字符");
}
return normalized;
}
} }

View File

@@ -3,5 +3,9 @@ package com.yoyuzh.auth;
public enum UserRole { public enum UserRole {
USER, USER,
MODERATOR, MODERATOR,
ADMIN ADMIN;
public boolean canAccessAdmin() {
return this == MODERATOR || this == ADMIN;
}
} }

View File

@@ -0,0 +1,44 @@
package com.yoyuzh.common.broker;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Deque;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "false", matchIfMissing = true)
public class InMemoryLightweightBrokerService implements LightweightBrokerService {
private final ConcurrentHashMap<String, Deque<Map<String, Object>>> queues = new ConcurrentHashMap<>();
@Override
public void publish(String topic, Map<String, Object> payload) {
queues.computeIfAbsent(topic, ignored -> new ConcurrentLinkedDeque<>())
.offerLast(copyPayload(payload));
}
@Override
public Optional<Map<String, Object>> poll(String topic) {
Deque<Map<String, Object>> queue = queues.get(topic);
if (queue == null) {
return Optional.empty();
}
Map<String, Object> payload = queue.pollFirst();
return payload == null ? Optional.empty() : Optional.of(new LinkedHashMap<>(payload));
}
@Override
public void requeue(String topic, Map<String, Object> payload) {
queues.computeIfAbsent(topic, ignored -> new ConcurrentLinkedDeque<>())
.offerFirst(copyPayload(payload));
}
private Map<String, Object> copyPayload(Map<String, Object> payload) {
return payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload);
}
}

View File

@@ -0,0 +1,13 @@
package com.yoyuzh.common.broker;
import java.util.Map;
import java.util.Optional;
public interface LightweightBrokerService {
void publish(String topic, Map<String, Object> payload);
Optional<Map<String, Object>> poll(String topic);
void requeue(String topic, Map<String, Object> payload);
}

View File

@@ -0,0 +1,87 @@
package com.yoyuzh.common.broker;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.config.AppRedisProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public class RedisLightweightBrokerService implements LightweightBrokerService {
private static final Logger log = LoggerFactory.getLogger(RedisLightweightBrokerService.class);
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final AppRedisProperties redisProperties;
public RedisLightweightBrokerService(StringRedisTemplate stringRedisTemplate,
ObjectMapper objectMapper,
AppRedisProperties redisProperties) {
this.stringRedisTemplate = stringRedisTemplate;
this.objectMapper = objectMapper;
this.redisProperties = redisProperties;
}
@Override
public void publish(String topic, Map<String, Object> payload) {
stringRedisTemplate.opsForList().rightPush(buildQueueKey(topic), toJson(payload));
}
@Override
public Optional<Map<String, Object>> poll(String topic) {
String queueKey = buildQueueKey(topic);
while (true) {
String payload = stringRedisTemplate.opsForList().leftPop(queueKey);
if (!StringUtils.hasText(payload)) {
return Optional.empty();
}
try {
return Optional.of(parsePayload(payload));
} catch (IllegalStateException ex) {
log.warn("Dropping malformed broker payload for topic {}", topic, ex);
}
}
}
@Override
public void requeue(String topic, Map<String, Object> payload) {
stringRedisTemplate.opsForList().leftPush(buildQueueKey(topic), toJson(payload));
}
private String buildQueueKey(String topic) {
return redisProperties.getKeyPrefix()
+ ":"
+ redisProperties.getNamespaces().getBroker()
+ ":"
+ topic
+ ":queue";
}
private String toJson(Map<String, Object> payload) {
try {
return objectMapper.writeValueAsString(payload == null ? Map.of() : payload);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to serialize broker payload", ex);
}
}
private Map<String, Object> parsePayload(String payload) {
try {
return objectMapper.readValue(payload, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to parse broker payload", ex);
}
}
}

View File

@@ -0,0 +1,32 @@
package com.yoyuzh.common.lock;
import java.time.Duration;
import java.util.function.Supplier;
public interface DistributedLockService {
<T> T executeWithLock(String lockName, Duration ttl, Supplier<T> action);
default void runWithLock(String lockName, Duration ttl, Runnable action) {
executeWithLock(lockName, ttl, () -> {
action.run();
return null;
});
}
static DistributedLockService noOp() {
return NoOpHolder.INSTANCE;
}
final class NoOpHolder {
private static final DistributedLockService INSTANCE = new DistributedLockService() {
@Override
public <T> T executeWithLock(String lockName, Duration ttl, Supplier<T> action) {
return action.get();
}
};
private NoOpHolder() {
}
}
}

View File

@@ -0,0 +1,17 @@
package com.yoyuzh.common.lock;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.util.function.Supplier;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "false", matchIfMissing = true)
public class NoOpDistributedLockService implements DistributedLockService {
@Override
public <T> T executeWithLock(String lockName, Duration ttl, Supplier<T> action) {
return action.get();
}
}

View File

@@ -0,0 +1,63 @@
package com.yoyuzh.common.lock;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.config.AppRedisProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.script.DefaultRedisScript;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.time.Duration;
import java.util.List;
import java.util.UUID;
import java.util.function.Supplier;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public class RedisDistributedLockService implements DistributedLockService {
private static final DefaultRedisScript<Long> RELEASE_SCRIPT = new DefaultRedisScript<>(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
Long.class
);
private final StringRedisTemplate stringRedisTemplate;
private final AppRedisProperties redisProperties;
public RedisDistributedLockService(StringRedisTemplate stringRedisTemplate,
AppRedisProperties redisProperties) {
this.stringRedisTemplate = stringRedisTemplate;
this.redisProperties = redisProperties;
}
@Override
public <T> T executeWithLock(String lockName, Duration ttl, Supplier<T> action) {
if (!StringUtils.hasText(lockName)) {
return action.get();
}
String key = buildLockKey(lockName);
String ownerToken = UUID.randomUUID().toString();
Duration effectiveTtl = ttl == null || ttl.isZero() || ttl.isNegative()
? Duration.ofSeconds(60)
: ttl;
Boolean acquired = stringRedisTemplate.opsForValue().setIfAbsent(key, ownerToken, effectiveTtl);
if (!Boolean.TRUE.equals(acquired)) {
throw new BusinessException(ErrorCode.UNKNOWN, "操作正在处理中,请稍后重试");
}
try {
return action.get();
} finally {
stringRedisTemplate.execute(RELEASE_SCRIPT, List.of(key), ownerToken);
}
}
private String buildLockKey(String lockName) {
return redisProperties.getKeyPrefix()
+ ":" + redisProperties.getNamespaces().getLocks()
+ ":" + lockName.trim();
}
}

View File

@@ -5,6 +5,7 @@ import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.IOException; import java.io.IOException;
@@ -18,6 +19,7 @@ public class AndroidReleaseService {
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final AndroidReleaseProperties androidReleaseProperties; private final AndroidReleaseProperties androidReleaseProperties;
@Cacheable(cacheNames = RedisCacheNames.ANDROID_RELEASE, key = "'latest'")
public AndroidReleaseResponse getLatestRelease() { public AndroidReleaseResponse getLatestRelease() {
AndroidReleaseMetadata metadata = loadReleaseMetadata(); AndroidReleaseMetadata metadata = loadReleaseMetadata();
return new AndroidReleaseResponse( return new AndroidReleaseResponse(

View File

@@ -0,0 +1,159 @@
package com.yoyuzh.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "app.redis")
public class AppRedisProperties {
private boolean enabled;
private String keyPrefix = "yoyuzh";
private long ttlBufferSeconds = 60;
private final Cache cache = new Cache();
private final Namespaces namespaces = new Namespaces();
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getKeyPrefix() {
return keyPrefix;
}
public void setKeyPrefix(String keyPrefix) {
this.keyPrefix = keyPrefix;
}
public long getTtlBufferSeconds() {
return ttlBufferSeconds;
}
public void setTtlBufferSeconds(long ttlBufferSeconds) {
this.ttlBufferSeconds = ttlBufferSeconds;
}
public Cache getCache() {
return cache;
}
public Namespaces getNamespaces() {
return namespaces;
}
public static class Cache {
private long filesListTtlSeconds = 60;
private long directoryVersionTtlSeconds = 3600;
private long adminSummaryTtlSeconds = 30;
private long storagePoliciesTtlSeconds = 300;
private long androidReleaseTtlSeconds = 60;
public long getFilesListTtlSeconds() {
return filesListTtlSeconds;
}
public void setFilesListTtlSeconds(long filesListTtlSeconds) {
this.filesListTtlSeconds = filesListTtlSeconds;
}
public long getDirectoryVersionTtlSeconds() {
return directoryVersionTtlSeconds;
}
public void setDirectoryVersionTtlSeconds(long directoryVersionTtlSeconds) {
this.directoryVersionTtlSeconds = directoryVersionTtlSeconds;
}
public long getAdminSummaryTtlSeconds() {
return adminSummaryTtlSeconds;
}
public void setAdminSummaryTtlSeconds(long adminSummaryTtlSeconds) {
this.adminSummaryTtlSeconds = adminSummaryTtlSeconds;
}
public long getStoragePoliciesTtlSeconds() {
return storagePoliciesTtlSeconds;
}
public void setStoragePoliciesTtlSeconds(long storagePoliciesTtlSeconds) {
this.storagePoliciesTtlSeconds = storagePoliciesTtlSeconds;
}
public long getAndroidReleaseTtlSeconds() {
return androidReleaseTtlSeconds;
}
public void setAndroidReleaseTtlSeconds(long androidReleaseTtlSeconds) {
this.androidReleaseTtlSeconds = androidReleaseTtlSeconds;
}
}
public static class Namespaces {
private String cache = "cache";
private String auth = "auth";
private String transferSessions = "transfer-sessions";
private String uploadState = "upload-state";
private String locks = "locks";
private String fileEvents = "file-events";
private String broker = "broker";
public String getCache() {
return cache;
}
public void setCache(String cache) {
this.cache = cache;
}
public String getAuth() {
return auth;
}
public void setAuth(String auth) {
this.auth = auth;
}
public String getTransferSessions() {
return transferSessions;
}
public void setTransferSessions(String transferSessions) {
this.transferSessions = transferSessions;
}
public String getUploadState() {
return uploadState;
}
public void setUploadState(String uploadState) {
this.uploadState = uploadState;
}
public String getLocks() {
return locks;
}
public void setLocks(String locks) {
this.locks = locks;
}
public String getFileEvents() {
return fileEvents;
}
public void setFileEvents(String fileEvents) {
this.fileEvents = fileEvents;
}
public String getBroker() {
return broker;
}
public void setBroker(String broker) {
this.broker = broker;
}
}
}

View File

@@ -1,6 +1,8 @@
package com.yoyuzh.config; package com.yoyuzh.config;
import com.yoyuzh.admin.AdminMetricsService; import com.yoyuzh.admin.AdminMetricsService;
import com.yoyuzh.auth.AuthClientType;
import com.yoyuzh.auth.AuthTokenInvalidationService;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.auth.JwtTokenProvider; import com.yoyuzh.auth.JwtTokenProvider;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
@@ -24,6 +26,7 @@ import java.io.IOException;
public class JwtAuthenticationFilter extends OncePerRequestFilter { public class JwtAuthenticationFilter extends OncePerRequestFilter {
private final JwtTokenProvider jwtTokenProvider; private final JwtTokenProvider jwtTokenProvider;
private final AuthTokenInvalidationService authTokenInvalidationService;
private final CustomUserDetailsService userDetailsService; private final CustomUserDetailsService userDetailsService;
private final AdminMetricsService adminMetricsService; private final AdminMetricsService adminMetricsService;
@@ -36,6 +39,15 @@ public class JwtAuthenticationFilter extends OncePerRequestFilter {
String token = header.substring(7); String token = header.substring(7);
if (jwtTokenProvider.validateToken(token) if (jwtTokenProvider.validateToken(token)
&& SecurityContextHolder.getContext().getAuthentication() == null) { && SecurityContextHolder.getContext().getAuthentication() == null) {
Long userId = jwtTokenProvider.getUserId(token);
AuthClientType clientType = jwtTokenProvider.getClientType(token);
if (authTokenInvalidationService.isAccessTokenRevoked(
userId,
clientType,
jwtTokenProvider.getIssuedAt(token))) {
filterChain.doFilter(request, response);
return;
}
String username = jwtTokenProvider.getUsername(token); String username = jwtTokenProvider.getUsername(token);
User domainUser; User domainUser;
try { try {

View File

@@ -0,0 +1,21 @@
package com.yoyuzh.config;
import java.util.Set;
public final class RedisCacheNames {
public static final String FILES_LIST = "files:list";
public static final String ADMIN_SUMMARY = "admin:summary";
public static final String STORAGE_POLICIES = "admin:storage-policies";
public static final String ANDROID_RELEASE = "android:release";
public static final Set<String> ALL = Set.of(
FILES_LIST,
ADMIN_SUMMARY,
STORAGE_POLICIES,
ANDROID_RELEASE
);
private RedisCacheNames() {
}
}

View File

@@ -0,0 +1,73 @@
package com.yoyuzh.config;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.support.NoOpCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.cache.RedisCacheConfiguration;
import org.springframework.data.redis.cache.RedisCacheManager;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.RedisSerializationContext;
import org.springframework.data.redis.serializer.StringRedisSerializer;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
@Configuration
@EnableCaching
public class RedisConfiguration {
@Bean
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public CacheManager redisCacheManager(RedisConnectionFactory redisConnectionFactory,
ObjectMapper objectMapper,
AppRedisProperties redisProperties) {
RedisCacheConfiguration baseConfiguration = RedisCacheConfiguration.defaultCacheConfig()
.computePrefixWith(cacheName -> redisProperties.getKeyPrefix()
+ ":" + redisProperties.getNamespaces().getCache()
+ ":" + cacheName + ":")
.serializeKeysWith(RedisSerializationContext.SerializationPair.fromSerializer(new StringRedisSerializer()))
.serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(
redisValueSerializer(objectMapper)))
.disableCachingNullValues();
Map<String, RedisCacheConfiguration> cacheConfigurations = new HashMap<>();
cacheConfigurations.put(
RedisCacheNames.FILES_LIST,
baseConfiguration.entryTtl(Duration.ofSeconds(redisProperties.getCache().getFilesListTtlSeconds()))
);
cacheConfigurations.put(
RedisCacheNames.ADMIN_SUMMARY,
baseConfiguration.entryTtl(Duration.ofSeconds(redisProperties.getCache().getAdminSummaryTtlSeconds()))
);
cacheConfigurations.put(
RedisCacheNames.STORAGE_POLICIES,
baseConfiguration.entryTtl(Duration.ofSeconds(redisProperties.getCache().getStoragePoliciesTtlSeconds()))
);
cacheConfigurations.put(
RedisCacheNames.ANDROID_RELEASE,
baseConfiguration.entryTtl(Duration.ofSeconds(redisProperties.getCache().getAndroidReleaseTtlSeconds()))
);
return RedisCacheManager.builder(redisConnectionFactory)
.cacheDefaults(baseConfiguration)
.withInitialCacheConfigurations(cacheConfigurations)
.initialCacheNames(RedisCacheNames.ALL)
.build();
}
@Bean
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "false", matchIfMissing = true)
public CacheManager noOpCacheManager() {
return new NoOpCacheManager();
}
static GenericJackson2JsonRedisSerializer redisValueSerializer(ObjectMapper objectMapper) {
return new GenericJackson2JsonRedisSerializer(objectMapper.copy());
}
}

View File

@@ -0,0 +1,78 @@
package com.yoyuzh.files.core;
import com.yoyuzh.auth.User;
import com.yoyuzh.files.policy.StoragePolicyCapabilities;
import com.yoyuzh.files.policy.StoragePolicyService;
import java.util.Optional;
final class ContentAssetBindingService {
private final FileEntityRepository fileEntityRepository;
private final StoredFileEntityRepository storedFileEntityRepository;
private final StoragePolicyService storagePolicyService;
ContentAssetBindingService(FileEntityRepository fileEntityRepository,
StoredFileEntityRepository storedFileEntityRepository,
StoragePolicyService storagePolicyService) {
this.fileEntityRepository = fileEntityRepository;
this.storedFileEntityRepository = storedFileEntityRepository;
this.storagePolicyService = storagePolicyService;
}
FileEntity createOrReferencePrimaryEntity(User user, FileBlob blob) {
if (fileEntityRepository == null) {
return createTransientPrimaryEntity(user, blob);
}
Optional<FileEntity> existingEntity = fileEntityRepository.findByObjectKeyAndEntityType(
blob.getObjectKey(),
FileEntityType.VERSION
);
if (existingEntity.isPresent()) {
FileEntity entity = existingEntity.get();
entity.setReferenceCount(entity.getReferenceCount() + 1);
return fileEntityRepository.save(entity);
}
return fileEntityRepository.save(createTransientPrimaryEntity(user, blob));
}
StoragePolicyCapabilities resolveDefaultStoragePolicyCapabilities() {
if (storagePolicyService == null) {
return null;
}
return storagePolicyService.readCapabilities(storagePolicyService.ensureDefaultPolicy());
}
void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) {
if (storedFileEntityRepository == null) {
return;
}
StoredFileEntity relation = new StoredFileEntity();
relation.setStoredFile(storedFile);
relation.setFileEntity(primaryEntity);
relation.setEntityRole("PRIMARY");
storedFileEntityRepository.save(relation);
}
private FileEntity createTransientPrimaryEntity(User user, FileBlob blob) {
FileEntity entity = new FileEntity();
entity.setObjectKey(blob.getObjectKey());
entity.setContentType(blob.getContentType());
entity.setSize(blob.getSize());
entity.setEntityType(FileEntityType.VERSION);
entity.setReferenceCount(1);
entity.setCreatedBy(user);
entity.setStoragePolicyId(resolveDefaultStoragePolicyId());
return entity;
}
private Long resolveDefaultStoragePolicyId() {
if (storagePolicyService == null) {
return null;
}
return storagePolicyService.ensureDefaultPolicy().getId();
}
}

View File

@@ -0,0 +1,103 @@
package com.yoyuzh.files.core;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.storage.FileContentStorage;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
final class ContentBlobLifecycleService {
private final StoredFileRepository storedFileRepository;
private final FileBlobRepository fileBlobRepository;
private final FileContentStorage fileContentStorage;
ContentBlobLifecycleService(StoredFileRepository storedFileRepository,
FileBlobRepository fileBlobRepository,
FileContentStorage fileContentStorage) {
this.storedFileRepository = storedFileRepository;
this.fileBlobRepository = fileBlobRepository;
this.fileContentStorage = fileContentStorage;
}
<T> T executeAfterBlobStored(String objectKey, Supplier<T> operation) {
try {
return operation.get();
} catch (RuntimeException ex) {
try {
fileContentStorage.deleteBlob(objectKey);
} catch (RuntimeException cleanupEx) {
ex.addSuppressed(cleanupEx);
}
throw ex;
}
}
void cleanupWrittenBlobs(List<String> writtenBlobObjectKeys, RuntimeException ex) {
for (String objectKey : writtenBlobObjectKeys) {
try {
fileContentStorage.deleteBlob(objectKey);
} catch (RuntimeException cleanupEx) {
ex.addSuppressed(cleanupEx);
}
}
}
FileBlob createAndSaveBlob(String objectKey, String contentType, long size) {
FileBlob blob = new FileBlob();
blob.setObjectKey(objectKey);
blob.setContentType(contentType);
blob.setSize(size);
return fileBlobRepository.save(blob);
}
FileBlob getRequiredBlob(StoredFile storedFile) {
if (storedFile.isDirectory() || storedFile.getBlob() == null) {
throw new BusinessException(ErrorCode.FILE_NOT_FOUND, "文件内容不存在");
}
return storedFile.getBlob();
}
List<FileBlob> collectBlobsToDelete(List<StoredFile> filesToDelete) {
Map<Long, BlobDeletionCandidate> candidates = new HashMap<>();
for (StoredFile file : filesToDelete) {
if (file.getBlob() == null || file.getBlob().getId() == null) {
continue;
}
BlobDeletionCandidate candidate = candidates.computeIfAbsent(
file.getBlob().getId(),
ignored -> new BlobDeletionCandidate(file.getBlob())
);
candidate.referencesToDelete += 1;
}
List<FileBlob> blobsToDelete = new ArrayList<>();
for (BlobDeletionCandidate candidate : candidates.values()) {
long currentReferences = storedFileRepository.countByBlobId(candidate.blob.getId());
if (currentReferences == candidate.referencesToDelete) {
blobsToDelete.add(candidate.blob);
}
}
return blobsToDelete;
}
void deleteBlobs(List<FileBlob> blobsToDelete) {
for (FileBlob blob : blobsToDelete) {
fileContentStorage.deleteBlob(blob.getObjectKey());
fileBlobRepository.delete(blob);
}
}
private static final class BlobDeletionCandidate {
private final FileBlob blob;
private long referencesToDelete;
private BlobDeletionCandidate(FileBlob blob) {
this.blob = blob;
}
}
}

View File

@@ -0,0 +1,75 @@
package com.yoyuzh.files.core;
import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import org.springframework.util.StringUtils;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
final class ExternalImportRulesService {
private final WorkspaceNodeRulesService workspaceNodeRulesService;
private final FileUploadRulesService fileUploadRulesService;
ExternalImportRulesService(WorkspaceNodeRulesService workspaceNodeRulesService,
FileUploadRulesService fileUploadRulesService) {
this.workspaceNodeRulesService = workspaceNodeRulesService;
this.fileUploadRulesService = fileUploadRulesService;
}
List<String> normalizeDirectories(List<String> directories) {
if (directories == null || directories.isEmpty()) {
return List.of();
}
return directories.stream()
.map(workspaceNodeRulesService::normalizeDirectoryPath)
.distinct()
.sorted(Comparator.comparingInt(String::length).thenComparing(String::compareTo))
.toList();
}
List<FileService.ExternalFileImport> normalizeFiles(List<FileService.ExternalFileImport> files) {
if (files == null || files.isEmpty()) {
return List.of();
}
return files.stream()
.map(file -> new FileService.ExternalFileImport(
workspaceNodeRulesService.normalizeDirectoryPath(file.path()),
workspaceNodeRulesService.normalizeLeafName(file.filename()),
StringUtils.hasText(file.contentType()) ? file.contentType().trim() : "application/octet-stream",
file.content() == null ? new byte[0] : file.content()
))
.toList();
}
void validateBatch(User recipient,
List<String> directories,
List<FileService.ExternalFileImport> files) {
fileUploadRulesService.ensureWithinStorageQuota(recipient, files.stream().mapToLong(FileService.ExternalFileImport::size).sum());
Set<String> plannedTargets = new LinkedHashSet<>();
for (String directory : directories) {
if ("/".equals(directory)) {
continue;
}
if (!plannedTargets.add(directory)) {
continue;
}
String parentPath = workspaceNodeRulesService.extractParentPath(directory);
String directoryName = workspaceNodeRulesService.extractLeafName(directory);
workspaceNodeRulesService.ensureNodeNameAvailable(recipient.getId(), parentPath, directoryName, "解压目标已存在");
}
for (FileService.ExternalFileImport file : files) {
String logicalPath = workspaceNodeRulesService.buildTargetLogicalPath(file.path(), file.filename());
if (plannedTargets.contains(logicalPath) || !plannedTargets.add(logicalPath)) {
throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在");
}
workspaceNodeRulesService.ensureNodeNameAvailable(recipient.getId(), file.path(), file.filename(), "同目录下文件已存在");
}
}
}

View File

@@ -3,12 +3,16 @@ package com.yoyuzh.files.core;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query; import org.springframework.data.jpa.repository.Query;
import java.util.Collection;
import java.util.List;
import java.util.Optional; import java.util.Optional;
public interface FileBlobRepository extends JpaRepository<FileBlob, Long> { public interface FileBlobRepository extends JpaRepository<FileBlob, Long> {
Optional<FileBlob> findByObjectKey(String objectKey); Optional<FileBlob> findByObjectKey(String objectKey);
List<FileBlob> findAllByObjectKeyIn(Collection<String> objectKeys);
@Query(""" @Query("""
select coalesce(sum(b.size), 0) select coalesce(sum(b.size), 0)
from FileBlob b from FileBlob b

View File

@@ -1,11 +1,18 @@
package com.yoyuzh.files.core; package com.yoyuzh.files.core;
import com.yoyuzh.api.v2.ApiV2Exception;
import com.yoyuzh.api.v2.shares.CreateShareV2Request;
import com.yoyuzh.api.v2.shares.ImportShareV2Request;
import com.yoyuzh.api.v2.shares.ShareV2Response;
import com.yoyuzh.auth.CustomUserDetailsService; import com.yoyuzh.auth.CustomUserDetailsService;
import com.yoyuzh.common.ApiResponse; import com.yoyuzh.common.ApiResponse;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.share.CreateFileShareLinkResponse; import com.yoyuzh.files.share.CreateFileShareLinkResponse;
import com.yoyuzh.files.share.FileShareDetailsResponse; import com.yoyuzh.files.share.FileShareDetailsResponse;
import com.yoyuzh.files.share.ImportSharedFileRequest; import com.yoyuzh.files.share.ImportSharedFileRequest;
import com.yoyuzh.files.share.ShareV2Service;
import com.yoyuzh.files.upload.CompleteUploadRequest; import com.yoyuzh.files.upload.CompleteUploadRequest;
import com.yoyuzh.files.upload.InitiateUploadRequest; import com.yoyuzh.files.upload.InitiateUploadRequest;
import com.yoyuzh.files.upload.InitiateUploadResponse; import com.yoyuzh.files.upload.InitiateUploadResponse;
@@ -37,6 +44,7 @@ public class FileController {
private final FileService fileService; private final FileService fileService;
private final CustomUserDetailsService userDetailsService; private final CustomUserDetailsService userDetailsService;
private final ShareV2Service shareV2Service;
@Operation(summary = "上传文件") @Operation(summary = "上传文件")
@PostMapping("/upload") @PostMapping("/upload")
@@ -148,15 +156,46 @@ public class FileController {
@PostMapping("/{fileId}/share-links") @PostMapping("/{fileId}/share-links")
public ApiResponse<CreateFileShareLinkResponse> createShareLink(@AuthenticationPrincipal UserDetails userDetails, public ApiResponse<CreateFileShareLinkResponse> createShareLink(@AuthenticationPrincipal UserDetails userDetails,
@PathVariable Long fileId) { @PathVariable Long fileId) {
return ApiResponse.success( try {
fileService.createShareLink(userDetailsService.loadDomainUser(userDetails.getUsername()), fileId) ShareV2Response response = shareV2Service.createShare(
userDetailsService.loadDomainUser(userDetails.getUsername()),
new CreateShareV2Request(fileId, null, null, null, null, null, null)
); );
if (response.file() == null) {
throw new BusinessException(ErrorCode.UNKNOWN, "share file metadata missing");
}
return ApiResponse.success(new CreateFileShareLinkResponse(
response.token(),
response.file().filename(),
response.file().size(),
response.file().contentType(),
response.createdAt()
));
} catch (ApiV2Exception ex) {
throw mapLegacyShareApiException(ex);
}
} }
@Operation(summary = "查看分享详情") @Operation(summary = "查看分享详情")
@GetMapping("/share-links/{token}") @GetMapping("/share-links/{token}")
public ApiResponse<FileShareDetailsResponse> getShareDetails(@PathVariable String token) { public ApiResponse<FileShareDetailsResponse> getShareDetails(@PathVariable String token) {
return ApiResponse.success(fileService.getShareDetails(token)); try {
ShareV2Response response = shareV2Service.getShare(token);
if (response.file() == null) {
throw new BusinessException(ErrorCode.PERMISSION_DENIED, "璇ュ垎浜摼鎺ラ渶瑕侀獙璇佸瘑鐮?");
}
return ApiResponse.success(new FileShareDetailsResponse(
response.token(),
response.ownerUsername(),
response.file().filename(),
response.file().size(),
response.file().contentType(),
response.file().directory(),
response.createdAt()
));
} catch (ApiV2Exception ex) {
throw mapLegacyShareApiException(ex);
}
} }
@Operation(summary = "导入共享文件") @Operation(summary = "导入共享文件")
@@ -164,13 +203,17 @@ public class FileController {
public ApiResponse<FileMetadataResponse> importSharedFile(@AuthenticationPrincipal UserDetails userDetails, public ApiResponse<FileMetadataResponse> importSharedFile(@AuthenticationPrincipal UserDetails userDetails,
@PathVariable String token, @PathVariable String token,
@Valid @RequestBody ImportSharedFileRequest request) { @Valid @RequestBody ImportSharedFileRequest request) {
try {
return ApiResponse.success( return ApiResponse.success(
fileService.importSharedFile( shareV2Service.importSharedFile(
userDetailsService.loadDomainUser(userDetails.getUsername()), userDetailsService.loadDomainUser(userDetails.getUsername()),
token, token,
request.path() new ImportShareV2Request(request.path(), null)
) )
); );
} catch (ApiV2Exception ex) {
throw mapLegacyShareApiException(ex);
}
} }
@Operation(summary = "删除文件") @Operation(summary = "删除文件")
@@ -190,4 +233,14 @@ public class FileController {
fileId fileId
)); ));
} }
private BusinessException mapLegacyShareApiException(ApiV2Exception ex) {
ErrorCode code = switch (ex.getErrorCode()) {
case FILE_NOT_FOUND -> ErrorCode.FILE_NOT_FOUND;
case PERMISSION_DENIED -> ErrorCode.PERMISSION_DENIED;
case NOT_LOGGED_IN -> ErrorCode.NOT_LOGGED_IN;
case BAD_REQUEST, INTERNAL_ERROR -> ErrorCode.UNKNOWN;
};
return new BusinessException(code, ex.getMessage());
}
} }

View File

@@ -0,0 +1,45 @@
package com.yoyuzh.files.core;
import com.yoyuzh.common.PageResponse;
import java.util.Collection;
import java.util.function.Supplier;
public interface FileListDirectoryCacheService {
PageResponse<FileMetadataResponse> getOrLoad(Long userId,
String path,
int page,
int size,
Supplier<PageResponse<FileMetadataResponse>> loader);
void touchDirectories(Long userId, Collection<String> paths);
default void touchDirectory(Long userId, String path) {
touchDirectories(userId, java.util.List.of(path));
}
static FileListDirectoryCacheService noOp() {
return NoOpHolder.INSTANCE;
}
final class NoOpHolder {
private static final FileListDirectoryCacheService INSTANCE = new FileListDirectoryCacheService() {
@Override
public PageResponse<FileMetadataResponse> getOrLoad(Long userId,
String path,
int page,
int size,
Supplier<PageResponse<FileMetadataResponse>> loader) {
return loader.get();
}
@Override
public void touchDirectories(Long userId, Collection<String> paths) {
}
};
private NoOpHolder() {
}
}
}

View File

@@ -5,6 +5,7 @@ import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.common.PageResponse; import com.yoyuzh.common.PageResponse;
import com.yoyuzh.common.lock.DistributedLockService;
import com.yoyuzh.config.FileStorageProperties; import com.yoyuzh.config.FileStorageProperties;
import com.yoyuzh.files.events.FileEventService; import com.yoyuzh.files.events.FileEventService;
import com.yoyuzh.files.events.FileEventType; import com.yoyuzh.files.events.FileEventType;
@@ -17,6 +18,7 @@ import com.yoyuzh.files.share.FileShareLink;
import com.yoyuzh.files.share.FileShareLinkRepository; import com.yoyuzh.files.share.FileShareLinkRepository;
import com.yoyuzh.files.storage.FileContentStorage; import com.yoyuzh.files.storage.FileContentStorage;
import com.yoyuzh.files.storage.PreparedUpload; import com.yoyuzh.files.storage.PreparedUpload;
import com.yoyuzh.files.tasks.MediaMetadataTaskBrokerPublisher;
import com.yoyuzh.files.upload.CompleteUploadRequest; import com.yoyuzh.files.upload.CompleteUploadRequest;
import com.yoyuzh.files.upload.InitiateUploadRequest; import com.yoyuzh.files.upload.InitiateUploadRequest;
import com.yoyuzh.files.upload.InitiateUploadResponse; import com.yoyuzh.files.upload.InitiateUploadResponse;
@@ -40,6 +42,7 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.time.Clock; import java.time.Clock;
import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.ArrayList; import java.util.ArrayList;
@@ -64,7 +67,6 @@ public class FileService {
private static final long RECYCLE_BIN_RETENTION_DAYS = 10L; private static final long RECYCLE_BIN_RETENTION_DAYS = 10L;
private final StoredFileRepository storedFileRepository; private final StoredFileRepository storedFileRepository;
private final FileBlobRepository fileBlobRepository;
private final FileEntityRepository fileEntityRepository; private final FileEntityRepository fileEntityRepository;
private final StoredFileEntityRepository storedFileEntityRepository; private final StoredFileEntityRepository storedFileEntityRepository;
private final FileContentStorage fileContentStorage; private final FileContentStorage fileContentStorage;
@@ -76,8 +78,19 @@ public class FileService {
private final String packageDownloadSecret; private final String packageDownloadSecret;
private final long packageDownloadTtlSeconds; private final long packageDownloadTtlSeconds;
private final Clock clock; private final Clock clock;
private final WorkspaceNodeRulesService workspaceNodeRulesService;
private final FileUploadRulesService fileUploadRulesService;
private final ExternalImportRulesService externalImportRulesService;
private final ContentAssetBindingService contentAssetBindingService;
private final ContentBlobLifecycleService contentBlobLifecycleService;
@Autowired(required = false) @Autowired(required = false)
private FileEventService fileEventService; private FileEventService fileEventService;
@Autowired(required = false)
private FileListDirectoryCacheService fileListDirectoryCacheService = FileListDirectoryCacheService.noOp();
@Autowired(required = false)
private DistributedLockService distributedLockService = DistributedLockService.noOp();
@Autowired(required = false)
private MediaMetadataTaskBrokerPublisher mediaMetadataTaskBrokerPublisher;
@Autowired @Autowired
public FileService(StoredFileRepository storedFileRepository, public FileService(StoredFileRepository storedFileRepository,
@@ -103,7 +116,6 @@ public class FileService {
FileStorageProperties properties, FileStorageProperties properties,
Clock clock) { Clock clock) {
this.storedFileRepository = storedFileRepository; this.storedFileRepository = storedFileRepository;
this.fileBlobRepository = fileBlobRepository;
this.fileEntityRepository = fileEntityRepository; this.fileEntityRepository = fileEntityRepository;
this.storedFileEntityRepository = storedFileEntityRepository; this.storedFileEntityRepository = storedFileEntityRepository;
this.fileContentStorage = fileContentStorage; this.fileContentStorage = fileContentStorage;
@@ -119,6 +131,11 @@ public class FileService {
: null; : null;
this.packageDownloadTtlSeconds = Math.max(1, properties.getS3().getPackageDownloadTtlSeconds()); this.packageDownloadTtlSeconds = Math.max(1, properties.getS3().getPackageDownloadTtlSeconds());
this.clock = clock; this.clock = clock;
this.workspaceNodeRulesService = new WorkspaceNodeRulesService(storedFileRepository, fileContentStorage);
this.fileUploadRulesService = new FileUploadRulesService(storedFileRepository, storagePolicyService, workspaceNodeRulesService, maxFileSize);
this.externalImportRulesService = new ExternalImportRulesService(workspaceNodeRulesService, fileUploadRulesService);
this.contentAssetBindingService = new ContentAssetBindingService(fileEntityRepository, storedFileEntityRepository, storagePolicyService);
this.contentBlobLifecycleService = new ContentBlobLifecycleService(storedFileRepository, fileBlobRepository, fileContentStorage);
} }
FileService(StoredFileRepository storedFileRepository, FileService(StoredFileRepository storedFileRepository,
@@ -144,13 +161,13 @@ public class FileService {
public FileMetadataResponse upload(User user, String path, MultipartFile multipartFile) { public FileMetadataResponse upload(User user, String path, MultipartFile multipartFile) {
String normalizedPath = normalizeDirectoryPath(path); String normalizedPath = normalizeDirectoryPath(path);
String filename = normalizeUploadFilename(multipartFile.getOriginalFilename()); String filename = normalizeUploadFilename(multipartFile.getOriginalFilename());
validateUpload(user, normalizedPath, filename, multipartFile.getSize()); fileUploadRulesService.validateUpload(user, normalizedPath, filename, multipartFile.getSize());
ensureDirectoryHierarchy(user, normalizedPath); ensureDirectoryHierarchy(user, normalizedPath);
String objectKey = createBlobObjectKey(); String objectKey = createBlobObjectKey();
return executeAfterBlobStored(objectKey, () -> { return contentBlobLifecycleService.executeAfterBlobStored(objectKey, () -> {
fileContentStorage.uploadBlob(objectKey, multipartFile); fileContentStorage.uploadBlob(objectKey, multipartFile);
FileBlob blob = createAndSaveBlob(objectKey, multipartFile.getContentType(), multipartFile.getSize()); FileBlob blob = contentBlobLifecycleService.createAndSaveBlob(objectKey, multipartFile.getContentType(), multipartFile.getSize());
return saveFileMetadata(user, normalizedPath, filename, multipartFile.getContentType(), multipartFile.getSize(), blob); return saveFileMetadata(user, normalizedPath, filename, multipartFile.getContentType(), multipartFile.getSize(), blob);
}); });
} }
@@ -158,10 +175,10 @@ public class FileService {
public InitiateUploadResponse initiateUpload(User user, InitiateUploadRequest request) { public InitiateUploadResponse initiateUpload(User user, InitiateUploadRequest request) {
String normalizedPath = normalizeDirectoryPath(request.path()); String normalizedPath = normalizeDirectoryPath(request.path());
String filename = normalizeLeafName(request.filename()); String filename = normalizeLeafName(request.filename());
validateUpload(user, normalizedPath, filename, request.size()); fileUploadRulesService.validateUpload(user, normalizedPath, filename, request.size());
String objectKey = createBlobObjectKey(); String objectKey = createBlobObjectKey();
StoragePolicyCapabilities capabilities = resolveDefaultStoragePolicyCapabilities(); StoragePolicyCapabilities capabilities = contentAssetBindingService.resolveDefaultStoragePolicyCapabilities();
if (capabilities != null && !capabilities.directUpload()) { if (capabilities != null && !capabilities.directUpload()) {
return new InitiateUploadResponse(false, "", "POST", Map.of(), objectKey); return new InitiateUploadResponse(false, "", "POST", Map.of(), objectKey);
} }
@@ -187,12 +204,12 @@ public class FileService {
String normalizedPath = normalizeDirectoryPath(request.path()); String normalizedPath = normalizeDirectoryPath(request.path());
String filename = normalizeLeafName(request.filename()); String filename = normalizeLeafName(request.filename());
String objectKey = normalizeBlobObjectKey(request.storageName()); String objectKey = normalizeBlobObjectKey(request.storageName());
validateUpload(user, normalizedPath, filename, request.size()); fileUploadRulesService.validateUpload(user, normalizedPath, filename, request.size());
ensureDirectoryHierarchy(user, normalizedPath); ensureDirectoryHierarchy(user, normalizedPath);
return executeAfterBlobStored(objectKey, () -> { return contentBlobLifecycleService.executeAfterBlobStored(objectKey, () -> {
fileContentStorage.completeBlobUpload(objectKey, request.contentType(), request.size()); fileContentStorage.completeBlobUpload(objectKey, request.contentType(), request.size());
FileBlob blob = createAndSaveBlob(objectKey, request.contentType(), request.size()); FileBlob blob = contentBlobLifecycleService.createAndSaveBlob(objectKey, request.contentType(), request.size());
return saveFileMetadata(user, normalizedPath, filename, request.contentType(), request.size(), blob); return saveFileMetadata(user, normalizedPath, filename, request.contentType(), request.size(), blob);
}); });
} }
@@ -205,9 +222,7 @@ public class FileService {
} }
String parentPath = extractParentPath(normalizedPath); String parentPath = extractParentPath(normalizedPath);
String directoryName = extractLeafName(normalizedPath); String directoryName = extractLeafName(normalizedPath);
if (storedFileRepository.existsByUserIdAndPathAndFilename(user.getId(), parentPath, directoryName)) { workspaceNodeRulesService.ensureNodeNameAvailable(user.getId(), parentPath, directoryName, "目录已存在");
throw new BusinessException(ErrorCode.UNKNOWN, "目录已存在");
}
fileContentStorage.createDirectory(user.getId(), normalizedPath); fileContentStorage.createDirectory(user.getId(), normalizedPath);
@@ -215,18 +230,23 @@ public class FileService {
storedFile.setUser(user); storedFile.setUser(user);
storedFile.setFilename(directoryName); storedFile.setFilename(directoryName);
storedFile.setPath(parentPath); storedFile.setPath(parentPath);
storedFile.setLegacyStorageName(directoryName);
storedFile.setContentType("directory"); storedFile.setContentType("directory");
storedFile.setSize(0L); storedFile.setSize(0L);
storedFile.setDirectory(true); storedFile.setDirectory(true);
return toResponse(storedFileRepository.save(storedFile)); FileMetadataResponse response = toResponse(storedFileRepository.save(storedFile));
touchDirectoryListings(user, parentPath);
return response;
} }
public PageResponse<FileMetadataResponse> list(User user, String path, int page, int size) { public PageResponse<FileMetadataResponse> list(User user, String path, int page, int size) {
String normalizedPath = normalizeDirectoryPath(path); String normalizedPath = normalizeDirectoryPath(path);
return fileListDirectoryCacheService.getOrLoad(user.getId(), normalizedPath, page, size, () -> {
Page<StoredFile> result = storedFileRepository.findByUserIdAndPathOrderByDirectoryDescCreatedAtDesc( Page<StoredFile> result = storedFileRepository.findByUserIdAndPathOrderByDirectoryDescCreatedAtDesc(
user.getId(), normalizedPath, PageRequest.of(page, size)); user.getId(), normalizedPath, PageRequest.of(page, size));
List<FileMetadataResponse> items = result.getContent().stream().map(this::toResponse).toList(); List<FileMetadataResponse> items = result.getContent().stream().map(this::toResponse).toList();
return new PageResponse<>(items, result.getTotalElements(), page, size); return new PageResponse<>(items, result.getTotalElements(), page, size);
});
} }
public List<FileMetadataResponse> recent(User user) { public List<FileMetadataResponse> recent(User user) {
@@ -244,8 +264,9 @@ public class FileService {
@Transactional @Transactional
public void ensureDefaultDirectories(User user) { public void ensureDefaultDirectories(User user) {
boolean createdAny = false;
for (String directoryName : DEFAULT_DIRECTORIES) { for (String directoryName : DEFAULT_DIRECTORIES) {
if (storedFileRepository.existsByUserIdAndPathAndFilename(user.getId(), "/", directoryName)) { if (workspaceNodeRulesService.existsNodeName(user.getId(), "/", directoryName)) {
continue; continue;
} }
@@ -256,10 +277,15 @@ public class FileService {
storedFile.setUser(user); storedFile.setUser(user);
storedFile.setFilename(directoryName); storedFile.setFilename(directoryName);
storedFile.setPath("/"); storedFile.setPath("/");
storedFile.setLegacyStorageName(directoryName);
storedFile.setContentType("directory"); storedFile.setContentType("directory");
storedFile.setSize(0L); storedFile.setSize(0L);
storedFile.setDirectory(true); storedFile.setDirectory(true);
storedFileRepository.save(storedFile); storedFileRepository.save(storedFile);
createdAny = true;
}
if (createdAny) {
touchDirectoryListings(user, "/");
} }
} }
@@ -275,20 +301,26 @@ public class FileService {
filesToRecycle.addAll(descendants); filesToRecycle.addAll(descendants);
} }
moveToRecycleBin(filesToRecycle, storedFile.getId()); moveToRecycleBin(filesToRecycle, storedFile.getId());
touchDirectoryListings(user, extractParentPath(fromPath));
recordFileEvent(user, FileEventType.DELETED, storedFile, fromPath, buildLogicalPath(storedFile)); recordFileEvent(user, FileEventType.DELETED, storedFile, fromPath, buildLogicalPath(storedFile));
} }
@Transactional @Transactional
public FileMetadataResponse restoreFromRecycleBin(User user, Long fileId) { public FileMetadataResponse restoreFromRecycleBin(User user, Long fileId) {
return distributedLockService.executeWithLock(
"files:recycle-restore:" + fileId,
Duration.ofSeconds(120),
() -> {
StoredFile recycleRoot = getOwnedRecycleRootFile(user, fileId); StoredFile recycleRoot = getOwnedRecycleRootFile(user, fileId);
String fromPath = buildLogicalPath(recycleRoot); String fromPath = buildLogicalPath(recycleRoot);
String toPath = buildTargetLogicalPath(requireRecycleOriginalPath(recycleRoot), recycleRoot.getFilename()); String restoreParentPath = requireRecycleOriginalPath(recycleRoot);
String toPath = buildTargetLogicalPath(restoreParentPath, recycleRoot.getFilename());
List<StoredFile> recycleGroupItems = loadRecycleGroupItems(recycleRoot); List<StoredFile> recycleGroupItems = loadRecycleGroupItems(recycleRoot);
long additionalBytes = recycleGroupItems.stream() long additionalBytes = recycleGroupItems.stream()
.filter(item -> !item.isDirectory()) .filter(item -> !item.isDirectory())
.mapToLong(StoredFile::getSize) .mapToLong(StoredFile::getSize)
.sum(); .sum();
ensureWithinStorageQuota(user, additionalBytes); fileUploadRulesService.ensureWithinStorageQuota(user, additionalBytes);
validateRecycleRestoreTargets(user.getId(), recycleGroupItems); validateRecycleRestoreTargets(user.getId(), recycleGroupItems);
ensureRecycleRestoreParentHierarchy(user, recycleRoot); ensureRecycleRestoreParentHierarchy(user, recycleRoot);
@@ -300,9 +332,12 @@ public class FileService {
item.setRecycleRoot(false); item.setRecycleRoot(false);
} }
storedFileRepository.saveAll(recycleGroupItems); storedFileRepository.saveAll(recycleGroupItems);
touchDirectoryListings(user, restoreParentPath);
recordFileEvent(user, FileEventType.RESTORED, recycleRoot, fromPath, toPath); recordFileEvent(user, FileEventType.RESTORED, recycleRoot, fromPath, toPath);
return toResponse(recycleRoot); return toResponse(recycleRoot);
} }
);
}
@Scheduled(fixedDelay = 60 * 60 * 1000L) @Scheduled(fixedDelay = 60 * 60 * 1000L)
@Transactional @Transactional
@@ -312,11 +347,11 @@ public class FileService {
return; return;
} }
List<FileBlob> blobsToDelete = collectBlobsToDelete( List<FileBlob> blobsToDelete = contentBlobLifecycleService.collectBlobsToDelete(
expiredItems.stream().filter(item -> !item.isDirectory()).toList() expiredItems.stream().filter(item -> !item.isDirectory()).toList()
); );
storedFileRepository.deleteAll(expiredItems); storedFileRepository.deleteAll(expiredItems);
deleteBlobs(blobsToDelete); contentBlobLifecycleService.deleteBlobs(blobsToDelete);
} }
@Transactional @Transactional
@@ -327,9 +362,7 @@ public class FileService {
if (sanitizedFilename.equals(storedFile.getFilename())) { if (sanitizedFilename.equals(storedFile.getFilename())) {
return toResponse(storedFile); return toResponse(storedFile);
} }
if (storedFileRepository.existsByUserIdAndPathAndFilename(user.getId(), storedFile.getPath(), sanitizedFilename)) { workspaceNodeRulesService.ensureNodeNameAvailable(user.getId(), storedFile.getPath(), sanitizedFilename, "同目录下文件已存在");
throw new BusinessException(ErrorCode.UNKNOWN, "同目录下文件已存在");
}
if (storedFile.isDirectory()) { if (storedFile.isDirectory()) {
String oldLogicalPath = buildLogicalPath(storedFile); String oldLogicalPath = buildLogicalPath(storedFile);
@@ -353,6 +386,7 @@ public class FileService {
storedFile.setFilename(sanitizedFilename); storedFile.setFilename(sanitizedFilename);
FileMetadataResponse response = toResponse(storedFileRepository.save(storedFile)); FileMetadataResponse response = toResponse(storedFileRepository.save(storedFile));
touchDirectoryListings(user, storedFile.getPath());
recordFileEvent(user, FileEventType.RENAMED, storedFile, fromPath, buildLogicalPath(storedFile)); recordFileEvent(user, FileEventType.RENAMED, storedFile, fromPath, buildLogicalPath(storedFile));
return response; return response;
} }
@@ -367,9 +401,7 @@ public class FileService {
} }
ensureExistingDirectoryPath(user.getId(), normalizedTargetPath); ensureExistingDirectoryPath(user.getId(), normalizedTargetPath);
if (storedFileRepository.existsByUserIdAndPathAndFilename(user.getId(), normalizedTargetPath, storedFile.getFilename())) { workspaceNodeRulesService.ensureNodeNameAvailable(user.getId(), normalizedTargetPath, storedFile.getFilename(), "目标目录已存在同名文件");
throw new BusinessException(ErrorCode.UNKNOWN, "目标目录已存在同名文件");
}
if (storedFile.isDirectory()) { if (storedFile.isDirectory()) {
String oldLogicalPath = buildLogicalPath(storedFile); String oldLogicalPath = buildLogicalPath(storedFile);
@@ -396,6 +428,7 @@ public class FileService {
storedFile.setPath(normalizedTargetPath); storedFile.setPath(normalizedTargetPath);
FileMetadataResponse response = toResponse(storedFileRepository.save(storedFile)); FileMetadataResponse response = toResponse(storedFileRepository.save(storedFile));
touchDirectoryListings(user, extractParentPath(fromPath), normalizedTargetPath);
recordFileEvent(user, FileEventType.MOVED, storedFile, fromPath, buildLogicalPath(storedFile)); recordFileEvent(user, FileEventType.MOVED, storedFile, fromPath, buildLogicalPath(storedFile));
return response; return response;
} }
@@ -405,13 +438,13 @@ public class FileService {
StoredFile storedFile = getOwnedActiveFile(user, fileId, "复制"); StoredFile storedFile = getOwnedActiveFile(user, fileId, "复制");
String normalizedTargetPath = normalizeDirectoryPath(nextPath); String normalizedTargetPath = normalizeDirectoryPath(nextPath);
ensureExistingDirectoryPath(user.getId(), normalizedTargetPath); ensureExistingDirectoryPath(user.getId(), normalizedTargetPath);
if (storedFileRepository.existsByUserIdAndPathAndFilename(user.getId(), normalizedTargetPath, storedFile.getFilename())) { workspaceNodeRulesService.ensureNodeNameAvailable(user.getId(), normalizedTargetPath, storedFile.getFilename(), "目标目录已存在同名文件");
throw new BusinessException(ErrorCode.UNKNOWN, "目标目录已存在同名文件");
}
if (!storedFile.isDirectory()) { if (!storedFile.isDirectory()) {
ensureWithinStorageQuota(user, storedFile.getSize()); fileUploadRulesService.ensureWithinStorageQuota(user, storedFile.getSize());
return toResponse(saveCopiedStoredFile(copyStoredFile(storedFile, user, normalizedTargetPath), user)); FileMetadataResponse response = toResponse(saveCopiedStoredFile(copyStoredFile(storedFile, user, normalizedTargetPath), user));
touchDirectoryListings(user, normalizedTargetPath);
return response;
} }
String oldLogicalPath = buildLogicalPath(storedFile); String oldLogicalPath = buildLogicalPath(storedFile);
@@ -425,7 +458,7 @@ public class FileService {
.filter(descendant -> !descendant.isDirectory()) .filter(descendant -> !descendant.isDirectory())
.mapToLong(StoredFile::getSize) .mapToLong(StoredFile::getSize)
.sum(); .sum();
ensureWithinStorageQuota(user, additionalBytes); fileUploadRulesService.ensureWithinStorageQuota(user, additionalBytes);
List<StoredFile> copiedEntries = new ArrayList<>(); List<StoredFile> copiedEntries = new ArrayList<>();
StoredFile copiedRoot = copyStoredFile(storedFile, user, normalizedTargetPath); StoredFile copiedRoot = copyStoredFile(storedFile, user, normalizedTargetPath);
@@ -438,9 +471,7 @@ public class FileService {
.thenComparing(StoredFile::getFilename)) .thenComparing(StoredFile::getFilename))
.forEach(descendant -> { .forEach(descendant -> {
String copiedPath = remapCopiedPath(descendant.getPath(), oldLogicalPath, newLogicalPath); String copiedPath = remapCopiedPath(descendant.getPath(), oldLogicalPath, newLogicalPath);
if (storedFileRepository.existsByUserIdAndPathAndFilename(user.getId(), copiedPath, descendant.getFilename())) { workspaceNodeRulesService.ensureNodeNameAvailable(user.getId(), copiedPath, descendant.getFilename(), "目标目录已存在同名文件");
throw new BusinessException(ErrorCode.UNKNOWN, "目标目录已存在同名文件");
}
copiedEntries.add(copyStoredFile(descendant, user, copiedPath)); copiedEntries.add(copyStoredFile(descendant, user, copiedPath));
}); });
@@ -452,6 +483,7 @@ public class FileService {
savedRoot = savedEntry; savedRoot = savedEntry;
} }
} }
touchDirectoryListings(user, normalizedTargetPath);
return toResponse(savedRoot == null ? copiedRoot : savedRoot); return toResponse(savedRoot == null ? copiedRoot : savedRoot);
} }
@@ -470,7 +502,7 @@ public class FileService {
if (fileContentStorage.supportsDirectDownload()) { if (fileContentStorage.supportsDirectDownload()) {
return ResponseEntity.status(302) return ResponseEntity.status(302)
.location(URI.create(fileContentStorage.createBlobDownloadUrl( .location(URI.create(fileContentStorage.createBlobDownloadUrl(
getRequiredBlob(storedFile).getObjectKey(), contentBlobLifecycleService.getRequiredBlob(storedFile).getObjectKey(),
storedFile.getFilename()))) storedFile.getFilename())))
.build(); .build();
} }
@@ -480,7 +512,7 @@ public class FileService {
"attachment; filename*=UTF-8''" + URLEncoder.encode(storedFile.getFilename(), StandardCharsets.UTF_8)) "attachment; filename*=UTF-8''" + URLEncoder.encode(storedFile.getFilename(), StandardCharsets.UTF_8))
.contentType(MediaType.parseMediaType( .contentType(MediaType.parseMediaType(
storedFile.getContentType() == null ? MediaType.APPLICATION_OCTET_STREAM_VALUE : storedFile.getContentType())) storedFile.getContentType() == null ? MediaType.APPLICATION_OCTET_STREAM_VALUE : storedFile.getContentType()))
.body(fileContentStorage.readBlob(getRequiredBlob(storedFile).getObjectKey())); .body(fileContentStorage.readBlob(contentBlobLifecycleService.getRequiredBlob(storedFile).getObjectKey()));
} }
public DownloadUrlResponse getDownloadUrl(User user, Long fileId) { public DownloadUrlResponse getDownloadUrl(User user, Long fileId) {
@@ -496,7 +528,7 @@ public class FileService {
if (fileContentStorage.supportsDirectDownload()) { if (fileContentStorage.supportsDirectDownload()) {
return new DownloadUrlResponse(fileContentStorage.createBlobDownloadUrl( return new DownloadUrlResponse(fileContentStorage.createBlobDownloadUrl(
getRequiredBlob(storedFile).getObjectKey(), contentBlobLifecycleService.getRequiredBlob(storedFile).getObjectKey(),
storedFile.getFilename() storedFile.getFilename()
)); ));
} }
@@ -553,7 +585,7 @@ public class FileService {
sourceFile.getFilename(), sourceFile.getFilename(),
sourceFile.getContentType(), sourceFile.getContentType(),
sourceFile.getSize(), sourceFile.getSize(),
getRequiredBlob(sourceFile) contentBlobLifecycleService.getRequiredBlob(sourceFile)
); );
} }
@@ -566,12 +598,12 @@ public class FileService {
byte[] content) { byte[] content) {
String normalizedPath = normalizeDirectoryPath(path); String normalizedPath = normalizeDirectoryPath(path);
String normalizedFilename = normalizeLeafName(filename); String normalizedFilename = normalizeLeafName(filename);
validateUpload(recipient, normalizedPath, normalizedFilename, size); fileUploadRulesService.validateUpload(recipient, normalizedPath, normalizedFilename, size);
ensureDirectoryHierarchy(recipient, normalizedPath); ensureDirectoryHierarchy(recipient, normalizedPath);
String objectKey = createBlobObjectKey(); String objectKey = createBlobObjectKey();
return executeAfterBlobStored(objectKey, () -> { return contentBlobLifecycleService.executeAfterBlobStored(objectKey, () -> {
fileContentStorage.storeBlob(objectKey, contentType, content); fileContentStorage.storeBlob(objectKey, contentType, content);
FileBlob blob = createAndSaveBlob(objectKey, contentType, size); FileBlob blob = contentBlobLifecycleService.createAndSaveBlob(objectKey, contentType, size);
return saveFileMetadata( return saveFileMetadata(
recipient, recipient,
@@ -596,9 +628,9 @@ public class FileService {
List<String> directories, List<String> directories,
List<ExternalFileImport> files, List<ExternalFileImport> files,
ExternalImportProgressListener progressListener) { ExternalImportProgressListener progressListener) {
List<String> normalizedDirectories = normalizeExternalImportDirectories(directories); List<String> normalizedDirectories = externalImportRulesService.normalizeDirectories(directories);
List<ExternalFileImport> normalizedFiles = normalizeExternalImportFiles(files); List<ExternalFileImport> normalizedFiles = externalImportRulesService.normalizeFiles(files);
validateExternalImportBatch(recipient, normalizedDirectories, normalizedFiles); externalImportRulesService.validateBatch(recipient, normalizedDirectories, normalizedFiles);
List<String> writtenBlobObjectKeys = new ArrayList<>(); List<String> writtenBlobObjectKeys = new ArrayList<>();
int totalDirectoryCount = normalizedDirectories.size(); int totalDirectoryCount = normalizedDirectories.size();
@@ -619,7 +651,7 @@ public class FileService {
processedDirectoryCount, totalDirectoryCount); processedDirectoryCount, totalDirectoryCount);
} }
} catch (RuntimeException ex) { } catch (RuntimeException ex) {
cleanupWrittenBlobs(writtenBlobObjectKeys, ex); contentBlobLifecycleService.cleanupWrittenBlobs(writtenBlobObjectKeys, ex);
throw ex; throw ex;
} }
} }
@@ -658,7 +690,7 @@ public class FileService {
} }
public ZipCompatibleArchive readZipCompatibleArchive(StoredFile source) { public ZipCompatibleArchive readZipCompatibleArchive(StoredFile source) {
byte[] archiveBytes = fileContentStorage.readBlob(getRequiredBlob(source).getObjectKey()); byte[] archiveBytes = fileContentStorage.readBlob(contentBlobLifecycleService.getRequiredBlob(source).getObjectKey());
try (ZipInputStream zipInputStream = new ZipInputStream( try (ZipInputStream zipInputStream = new ZipInputStream(
new ByteArrayInputStream(archiveBytes), new ByteArrayInputStream(archiveBytes),
StandardCharsets.UTF_8)) { StandardCharsets.UTF_8)) {
@@ -709,7 +741,7 @@ public class FileService {
} }
private String buildPublicPackageDownloadUrl(StoredFile storedFile) { private String buildPublicPackageDownloadUrl(StoredFile storedFile) {
FileBlob blob = getRequiredBlob(storedFile); FileBlob blob = contentBlobLifecycleService.getRequiredBlob(storedFile);
String base = packageDownloadBaseUrl.endsWith("/") String base = packageDownloadBaseUrl.endsWith("/")
? packageDownloadBaseUrl.substring(0, packageDownloadBaseUrl.length() - 1) ? packageDownloadBaseUrl.substring(0, packageDownloadBaseUrl.length() - 1)
: packageDownloadBaseUrl; : packageDownloadBaseUrl;
@@ -817,68 +849,30 @@ public class FileService {
storedFile.setSize(size); storedFile.setSize(size);
storedFile.setDirectory(false); storedFile.setDirectory(false);
storedFile.setBlob(blob); storedFile.setBlob(blob);
storedFile.setLegacyStorageName(blob.getObjectKey());
FileEntity primaryEntity = createOrReferencePrimaryEntity(user, blob); FileEntity primaryEntity = createOrReferencePrimaryEntity(user, blob);
storedFile.setPrimaryEntity(primaryEntity); storedFile.setPrimaryEntity(primaryEntity);
StoredFile savedFile = storedFileRepository.save(storedFile); StoredFile savedFile = storedFileRepository.save(storedFile);
savePrimaryEntityRelation(savedFile, primaryEntity); savePrimaryEntityRelation(savedFile, primaryEntity);
touchDirectoryListings(user, normalizedPath);
publishMediaMetadataTrigger(savedFile);
recordFileEvent(user, FileEventType.CREATED, savedFile, null, buildLogicalPath(savedFile)); recordFileEvent(user, FileEventType.CREATED, savedFile, null, buildLogicalPath(savedFile));
return toResponse(savedFile); return toResponse(savedFile);
} }
private FileEntity createOrReferencePrimaryEntity(User user, FileBlob blob) { private FileEntity createOrReferencePrimaryEntity(User user, FileBlob blob) {
if (fileEntityRepository == null) { return contentAssetBindingService.createOrReferencePrimaryEntity(user, blob);
return createTransientPrimaryEntity(user, blob);
}
Optional<FileEntity> existingEntity = fileEntityRepository.findByObjectKeyAndEntityType(
blob.getObjectKey(),
FileEntityType.VERSION
);
if (existingEntity.isPresent()) {
FileEntity entity = existingEntity.get();
entity.setReferenceCount(entity.getReferenceCount() + 1);
return fileEntityRepository.save(entity);
}
return fileEntityRepository.save(createTransientPrimaryEntity(user, blob));
}
private FileEntity createTransientPrimaryEntity(User user, FileBlob blob) {
FileEntity entity = new FileEntity();
entity.setObjectKey(blob.getObjectKey());
entity.setContentType(blob.getContentType());
entity.setSize(blob.getSize());
entity.setEntityType(FileEntityType.VERSION);
entity.setReferenceCount(1);
entity.setCreatedBy(user);
entity.setStoragePolicyId(resolveDefaultStoragePolicyId());
return entity;
}
private Long resolveDefaultStoragePolicyId() {
if (storagePolicyService == null) {
return null;
}
return storagePolicyService.ensureDefaultPolicy().getId();
}
private StoragePolicyCapabilities resolveDefaultStoragePolicyCapabilities() {
if (storagePolicyService == null) {
return null;
}
return storagePolicyService.readCapabilities(storagePolicyService.ensureDefaultPolicy());
} }
private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) { private void savePrimaryEntityRelation(StoredFile storedFile, FileEntity primaryEntity) {
if (storedFileEntityRepository == null) { contentAssetBindingService.savePrimaryEntityRelation(storedFile, primaryEntity);
return;
} }
StoredFileEntity relation = new StoredFileEntity(); private void publishMediaMetadataTrigger(StoredFile storedFile) {
relation.setStoredFile(storedFile); if (mediaMetadataTaskBrokerPublisher == null) {
relation.setFileEntity(primaryEntity); return;
relation.setEntityRole("PRIMARY"); }
storedFileEntityRepository.save(relation); mediaMetadataTaskBrokerPublisher.publishAfterCommit(storedFile);
} }
private FileShareLink getShareLink(String token) { private FileShareLink getShareLink(String token) {
@@ -939,6 +933,10 @@ public class FileService {
} }
private void validateUpload(User user, String normalizedPath, String filename, long size) { private void validateUpload(User user, String normalizedPath, String filename, long size) {
if (fileUploadRulesService != null) {
fileUploadRulesService.validateUpload(user, normalizedPath, filename, size);
return;
}
long effectiveMaxUploadSize = Math.min(maxFileSize, user.getMaxUploadSizeBytes()); long effectiveMaxUploadSize = Math.min(maxFileSize, user.getMaxUploadSizeBytes());
StoragePolicy defaultPolicy = storagePolicyService == null ? null : storagePolicyService.ensureDefaultPolicy(); StoragePolicy defaultPolicy = storagePolicyService == null ? null : storagePolicyService.ensureDefaultPolicy();
StoragePolicyCapabilities capabilities = defaultPolicy == null ? null : storagePolicyService.readCapabilities(defaultPolicy); StoragePolicyCapabilities capabilities = defaultPolicy == null ? null : storagePolicyService.readCapabilities(defaultPolicy);
@@ -951,9 +949,7 @@ public class FileService {
if (size > effectiveMaxUploadSize) { if (size > effectiveMaxUploadSize) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件大小超出限制"); throw new BusinessException(ErrorCode.UNKNOWN, "文件大小超出限制");
} }
if (storedFileRepository.existsByUserIdAndPathAndFilename(user.getId(), normalizedPath, filename)) { workspaceNodeRulesService.ensureNodeNameAvailable(user.getId(), normalizedPath, filename, "同目录下文件已存在");
throw new BusinessException(ErrorCode.UNKNOWN, "同目录下文件已存在");
}
ensureWithinStorageQuota(user, size); ensureWithinStorageQuota(user, size);
} }
@@ -985,7 +981,7 @@ public class FileService {
private void validateExternalImportBatch(User recipient, private void validateExternalImportBatch(User recipient,
List<String> directories, List<String> directories,
List<ExternalFileImport> files) { List<ExternalFileImport> files) {
ensureWithinStorageQuota(recipient, files.stream().mapToLong(ExternalFileImport::size).sum()); fileUploadRulesService.ensureWithinStorageQuota(recipient, files.stream().mapToLong(ExternalFileImport::size).sum());
Set<String> plannedTargets = new LinkedHashSet<>(); Set<String> plannedTargets = new LinkedHashSet<>();
for (String directory : directories) { for (String directory : directories) {
@@ -997,9 +993,7 @@ public class FileService {
} }
String parentPath = extractParentPath(directory); String parentPath = extractParentPath(directory);
String directoryName = extractLeafName(directory); String directoryName = extractLeafName(directory);
if (storedFileRepository.existsByUserIdAndPathAndFilename(recipient.getId(), parentPath, directoryName)) { workspaceNodeRulesService.ensureNodeNameAvailable(recipient.getId(), parentPath, directoryName, "解压目标已存在");
throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在");
}
} }
for (ExternalFileImport file : files) { for (ExternalFileImport file : files) {
@@ -1007,13 +1001,15 @@ public class FileService {
if (plannedTargets.contains(logicalPath) || !plannedTargets.add(logicalPath)) { if (plannedTargets.contains(logicalPath) || !plannedTargets.add(logicalPath)) {
throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在"); throw new BusinessException(ErrorCode.UNKNOWN, "解压目标已存在");
} }
if (storedFileRepository.existsByUserIdAndPathAndFilename(recipient.getId(), file.path(), file.filename())) { workspaceNodeRulesService.ensureNodeNameAvailable(recipient.getId(), file.path(), file.filename(), "同目录下文件已存在");
throw new BusinessException(ErrorCode.UNKNOWN, "同目录下文件已存在");
}
} }
} }
private void ensureWithinStorageQuota(User user, long additionalBytes) { private void ensureWithinStorageQuota(User user, long additionalBytes) {
if (fileUploadRulesService != null) {
fileUploadRulesService.ensureWithinStorageQuota(user, additionalBytes);
return;
}
if (additionalBytes <= 0) { if (additionalBytes <= 0) {
return; return;
} }
@@ -1026,48 +1022,18 @@ public class FileService {
} }
private void ensureDirectoryHierarchy(User user, String normalizedPath) { private void ensureDirectoryHierarchy(User user, String normalizedPath) {
if ("/".equals(normalizedPath)) { workspaceNodeRulesService.ensureDirectoryHierarchy(user, normalizedPath);
return;
}
String[] segments = normalizedPath.substring(1).split("/");
String currentPath = "/";
for (String segment : segments) {
Optional<StoredFile> existing = storedFileRepository.findByUserIdAndPathAndFilename(user.getId(), currentPath, segment);
if (existing.isPresent()) {
if (!existing.get().isDirectory()) {
throw new BusinessException(ErrorCode.UNKNOWN, "目标路径不是目录");
}
currentPath = "/".equals(currentPath) ? "/" + segment : currentPath + "/" + segment;
continue;
}
String logicalPath = "/".equals(currentPath) ? "/" + segment : currentPath + "/" + segment;
fileContentStorage.ensureDirectory(user.getId(), logicalPath);
StoredFile storedFile = new StoredFile();
storedFile.setUser(user);
storedFile.setFilename(segment);
storedFile.setPath(currentPath);
storedFile.setContentType("directory");
storedFile.setSize(0L);
storedFile.setDirectory(true);
storedFileRepository.save(storedFile);
currentPath = logicalPath;
}
} }
private void storeExternalImportFile(User recipient, private void storeExternalImportFile(User recipient,
ExternalFileImport file, ExternalFileImport file,
List<String> writtenBlobObjectKeys) { List<String> writtenBlobObjectKeys) {
validateUpload(recipient, file.path(), file.filename(), file.size()); fileUploadRulesService.validateUpload(recipient, file.path(), file.filename(), file.size());
ensureDirectoryHierarchy(recipient, file.path()); ensureDirectoryHierarchy(recipient, file.path());
String objectKey = createBlobObjectKey(); String objectKey = createBlobObjectKey();
writtenBlobObjectKeys.add(objectKey); writtenBlobObjectKeys.add(objectKey);
fileContentStorage.storeBlob(objectKey, file.contentType(), file.content()); fileContentStorage.storeBlob(objectKey, file.contentType(), file.content());
FileBlob blob = createAndSaveBlob(objectKey, file.contentType(), file.size()); FileBlob blob = contentBlobLifecycleService.createAndSaveBlob(objectKey, file.contentType(), file.size());
saveFileMetadata( saveFileMetadata(
recipient, recipient,
file.path(), file.path(),
@@ -1130,12 +1096,7 @@ public class FileService {
} }
private void validateRecycleRestoreTargets(Long userId, List<StoredFile> recycleGroupItems) { private void validateRecycleRestoreTargets(Long userId, List<StoredFile> recycleGroupItems) {
for (StoredFile item : recycleGroupItems) { workspaceNodeRulesService.validateRecycleRestoreTargets(userId, recycleGroupItems, this::requireRecycleOriginalPath);
String originalPath = requireRecycleOriginalPath(item);
if (storedFileRepository.existsByUserIdAndPathAndFilename(userId, originalPath, item.getFilename())) {
throw new BusinessException(ErrorCode.UNKNOWN, "原目录已存在同名文件,无法恢复");
}
}
} }
private void ensureRecycleRestoreParentHierarchy(User user, StoredFile recycleRoot) { private void ensureRecycleRestoreParentHierarchy(User user, StoredFile recycleRoot) {
@@ -1143,28 +1104,11 @@ public class FileService {
} }
private void ensureExistingDirectoryPath(Long userId, String normalizedPath) { private void ensureExistingDirectoryPath(Long userId, String normalizedPath) {
if ("/".equals(normalizedPath)) { workspaceNodeRulesService.ensureExistingDirectoryPath(userId, normalizedPath);
return;
}
String[] segments = normalizedPath.substring(1).split("/");
String currentPath = "/";
for (String segment : segments) {
StoredFile directory = storedFileRepository.findByUserIdAndPathAndFilename(userId, currentPath, segment)
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "目标目录不存在"));
if (!directory.isDirectory()) {
throw new BusinessException(ErrorCode.UNKNOWN, "目标路径不是目录");
}
currentPath = "/".equals(currentPath) ? "/" + segment : currentPath + "/" + segment;
}
} }
private String normalizeUploadFilename(String originalFilename) { private String normalizeUploadFilename(String originalFilename) {
String filename = StringUtils.cleanPath(originalFilename); return workspaceNodeRulesService.normalizeUploadFilename(originalFilename);
if (!StringUtils.hasText(filename)) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件名不能为空");
}
return normalizeLeafName(filename);
} }
private FileMetadataResponse toResponse(StoredFile storedFile) { private FileMetadataResponse toResponse(StoredFile storedFile) {
@@ -1180,30 +1124,15 @@ public class FileService {
} }
private String normalizeDirectoryPath(String path) { private String normalizeDirectoryPath(String path) {
if (!StringUtils.hasText(path) || "/".equals(path.trim())) { return workspaceNodeRulesService.normalizeDirectoryPath(path);
return "/";
}
String normalized = path.replace("\\", "/").trim();
if (!normalized.startsWith("/")) {
normalized = "/" + normalized;
}
normalized = normalized.replaceAll("/{2,}", "/");
if (normalized.contains("..")) {
throw new BusinessException(ErrorCode.UNKNOWN, "路径不合法");
}
if (normalized.endsWith("/") && normalized.length() > 1) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
} }
private String extractParentPath(String normalizedPath) { private String extractParentPath(String normalizedPath) {
int lastSlash = normalizedPath.lastIndexOf('/'); return workspaceNodeRulesService.extractParentPath(normalizedPath);
return lastSlash <= 0 ? "/" : normalizedPath.substring(0, lastSlash);
} }
private String extractLeafName(String normalizedPath) { private String extractLeafName(String normalizedPath) {
return normalizedPath.substring(normalizedPath.lastIndexOf('/') + 1); return workspaceNodeRulesService.extractLeafName(normalizedPath);
} }
private String buildLogicalPath(StoredFile storedFile) { private String buildLogicalPath(StoredFile storedFile) {
@@ -1213,9 +1142,7 @@ public class FileService {
} }
private String buildTargetLogicalPath(String normalizedTargetPath, String filename) { private String buildTargetLogicalPath(String normalizedTargetPath, String filename) {
return "/".equals(normalizedTargetPath) return workspaceNodeRulesService.buildTargetLogicalPath(normalizedTargetPath, filename);
? "/" + filename
: normalizedTargetPath + "/" + filename;
} }
private String remapCopiedPath(String currentPath, String oldLogicalPath, String newLogicalPath) { private String remapCopiedPath(String currentPath, String oldLogicalPath, String newLogicalPath) {
@@ -1277,7 +1204,7 @@ public class FileService {
ArchiveBuildProgressState progressState) throws IOException { ArchiveBuildProgressState progressState) throws IOException {
ensureParentDirectoryEntries(zipOutputStream, createdEntries, entryName, progressState); ensureParentDirectoryEntries(zipOutputStream, createdEntries, entryName, progressState);
writeFileEntry(zipOutputStream, createdEntries, entryName, progressState, writeFileEntry(zipOutputStream, createdEntries, entryName, progressState,
fileContentStorage.readBlob(getRequiredBlob(file).getObjectKey())); fileContentStorage.readBlob(contentBlobLifecycleService.getRequiredBlob(file).getObjectKey()));
} }
private String buildZipEntryName(String rootDirectoryName, String rootLogicalPath, StoredFile storedFile) { private String buildZipEntryName(String rootDirectoryName, String rootLogicalPath, StoredFile storedFile) {
@@ -1476,15 +1403,26 @@ public class FileService {
fileEventService.record(user, eventType, storedFile.getId(), fromPath, toPath, payload); fileEventService.record(user, eventType, storedFile.getId(), fromPath, toPath, payload);
} }
private void touchDirectoryListings(User user, String... paths) {
if (user == null || user.getId() == null || paths == null || paths.length == 0) {
return;
}
List<String> affectedPaths = new ArrayList<>();
for (String path : paths) {
if (StringUtils.hasText(path)) {
affectedPaths.add(normalizeDirectoryPath(path));
}
}
if (affectedPaths.isEmpty()) {
return;
}
fileListDirectoryCacheService.touchDirectories(user.getId(), affectedPaths);
}
private String normalizeLeafName(String filename) { private String normalizeLeafName(String filename) {
String cleaned = StringUtils.cleanPath(filename == null ? "" : filename).trim(); return workspaceNodeRulesService.normalizeLeafName(filename);
if (!StringUtils.hasText(cleaned)) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件名不能为空");
}
if (cleaned.contains("/") || cleaned.contains("\\") || cleaned.contains("..")) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件名不合法");
}
return cleaned;
} }
private String createBlobObjectKey() { private String createBlobObjectKey() {
@@ -1499,37 +1437,6 @@ public class FileService {
return cleaned; return cleaned;
} }
private <T> T executeAfterBlobStored(String objectKey, BlobWriteOperation<T> operation) {
try {
return operation.run();
} catch (RuntimeException ex) {
try {
fileContentStorage.deleteBlob(objectKey);
} catch (RuntimeException cleanupEx) {
ex.addSuppressed(cleanupEx);
}
throw ex;
}
}
private void cleanupWrittenBlobs(List<String> writtenBlobObjectKeys, RuntimeException ex) {
for (String objectKey : writtenBlobObjectKeys) {
try {
fileContentStorage.deleteBlob(objectKey);
} catch (RuntimeException cleanupEx) {
ex.addSuppressed(cleanupEx);
}
}
}
private FileBlob createAndSaveBlob(String objectKey, String contentType, long size) {
FileBlob blob = new FileBlob();
blob.setObjectKey(objectKey);
blob.setContentType(contentType);
blob.setSize(size);
return fileBlobRepository.save(blob);
}
private FileMetadataResponse importReferencedBlob(User recipient, private FileMetadataResponse importReferencedBlob(User recipient,
String path, String path,
String filename, String filename,
@@ -1538,7 +1445,7 @@ public class FileService {
FileBlob blob) { FileBlob blob) {
String normalizedPath = normalizeDirectoryPath(path); String normalizedPath = normalizeDirectoryPath(path);
String normalizedFilename = normalizeLeafName(filename); String normalizedFilename = normalizeLeafName(filename);
validateUpload(recipient, normalizedPath, normalizedFilename, size); fileUploadRulesService.validateUpload(recipient, normalizedPath, normalizedFilename, size);
ensureDirectoryHierarchy(recipient, normalizedPath); ensureDirectoryHierarchy(recipient, normalizedPath);
return saveFileMetadata( return saveFileMetadata(
recipient, recipient,
@@ -1557,50 +1464,6 @@ public class FileService {
return storedFile.getBlob(); return storedFile.getBlob();
} }
private List<FileBlob> collectBlobsToDelete(List<StoredFile> filesToDelete) {
Map<Long, BlobDeletionCandidate> candidates = new HashMap<>();
for (StoredFile file : filesToDelete) {
if (file.getBlob() == null || file.getBlob().getId() == null) {
continue;
}
BlobDeletionCandidate candidate = candidates.computeIfAbsent(
file.getBlob().getId(),
ignored -> new BlobDeletionCandidate(file.getBlob())
);
candidate.referencesToDelete += 1;
}
List<FileBlob> blobsToDelete = new ArrayList<>();
for (BlobDeletionCandidate candidate : candidates.values()) {
long currentReferences = storedFileRepository.countByBlobId(candidate.blob.getId());
if (currentReferences == candidate.referencesToDelete) {
blobsToDelete.add(candidate.blob);
}
}
return blobsToDelete;
}
private void deleteBlobs(List<FileBlob> blobsToDelete) {
for (FileBlob blob : blobsToDelete) {
fileContentStorage.deleteBlob(blob.getObjectKey());
fileBlobRepository.delete(blob);
}
}
private static final class BlobDeletionCandidate {
private final FileBlob blob;
private long referencesToDelete;
private BlobDeletionCandidate(FileBlob blob) {
this.blob = blob;
}
}
@FunctionalInterface
private interface BlobWriteOperation<T> {
T run();
}
public static record ZipCompatibleArchive(List<ZipCompatibleArchiveEntry> entries, String commonRootDirectoryName) { public static record ZipCompatibleArchive(List<ZipCompatibleArchiveEntry> entries, String commonRootDirectoryName) {
} }

View File

@@ -0,0 +1,55 @@
package com.yoyuzh.files.core;
import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyCapabilities;
import com.yoyuzh.files.policy.StoragePolicyService;
public final class FileUploadRulesService {
private final StoredFileRepository storedFileRepository;
private final StoragePolicyService storagePolicyService;
private final WorkspaceNodeRulesService workspaceNodeRulesService;
private final long maxFileSize;
public FileUploadRulesService(StoredFileRepository storedFileRepository,
StoragePolicyService storagePolicyService,
WorkspaceNodeRulesService workspaceNodeRulesService,
long maxFileSize) {
this.storedFileRepository = storedFileRepository;
this.storagePolicyService = storagePolicyService;
this.workspaceNodeRulesService = workspaceNodeRulesService;
this.maxFileSize = maxFileSize;
}
public void validateUpload(User user, String normalizedPath, String filename, long size) {
long effectiveMaxUploadSize = Math.min(maxFileSize, user.getMaxUploadSizeBytes());
StoragePolicy defaultPolicy = storagePolicyService == null ? null : storagePolicyService.ensureDefaultPolicy();
StoragePolicyCapabilities capabilities = defaultPolicy == null ? null : storagePolicyService.readCapabilities(defaultPolicy);
if (defaultPolicy != null && defaultPolicy.getMaxSizeBytes() > 0) {
effectiveMaxUploadSize = Math.min(effectiveMaxUploadSize, defaultPolicy.getMaxSizeBytes());
}
if (capabilities != null && capabilities.maxObjectSize() > 0) {
effectiveMaxUploadSize = Math.min(effectiveMaxUploadSize, capabilities.maxObjectSize());
}
if (size > effectiveMaxUploadSize) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件大小超出限制");
}
workspaceNodeRulesService.ensureNodeNameAvailable(user.getId(), normalizedPath, filename, "同目录下文件已存在");
ensureWithinStorageQuota(user, size);
}
public void ensureWithinStorageQuota(User user, long additionalBytes) {
if (additionalBytes <= 0) {
return;
}
long usedBytes = storedFileRepository.sumFileSizeByUserId(user.getId());
long quotaBytes = user.getStorageQuotaBytes();
if (usedBytes > Long.MAX_VALUE - additionalBytes || usedBytes + additionalBytes > quotaBytes) {
throw new BusinessException(ErrorCode.UNKNOWN, "存储空间不足");
}
}
}

View File

@@ -0,0 +1,26 @@
package com.yoyuzh.files.core;
import com.yoyuzh.common.PageResponse;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.Collection;
import java.util.function.Supplier;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "false", matchIfMissing = true)
public class NoOpFileListDirectoryCacheService implements FileListDirectoryCacheService {
@Override
public PageResponse<FileMetadataResponse> getOrLoad(Long userId,
String path,
int page,
int size,
Supplier<PageResponse<FileMetadataResponse>> loader) {
return loader.get();
}
@Override
public void touchDirectories(Long userId, Collection<String> paths) {
}
}

View File

@@ -0,0 +1,166 @@
package com.yoyuzh.files.core;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.common.PageResponse;
import com.yoyuzh.config.AppRedisProperties;
import com.yoyuzh.config.RedisCacheNames;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Base64;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Supplier;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public class RedisFileListDirectoryCacheService implements FileListDirectoryCacheService {
private static final String SORT_CONTEXT = "directory-desc-created-desc";
private final CacheManager cacheManager;
private final StringRedisTemplate stringRedisTemplate;
private final AppRedisProperties redisProperties;
private final ObjectMapper objectMapper;
public RedisFileListDirectoryCacheService(CacheManager cacheManager,
StringRedisTemplate stringRedisTemplate,
AppRedisProperties redisProperties,
ObjectMapper objectMapper) {
this.cacheManager = cacheManager;
this.stringRedisTemplate = stringRedisTemplate;
this.redisProperties = redisProperties;
this.objectMapper = objectMapper;
}
@Override
public PageResponse<FileMetadataResponse> getOrLoad(Long userId,
String path,
int page,
int size,
Supplier<PageResponse<FileMetadataResponse>> loader) {
Cache cache = cacheManager.getCache(RedisCacheNames.FILES_LIST);
if (cache == null) {
return loader.get();
}
long version = readDirectoryVersion(userId, path);
String cacheKey = buildCacheKey(userId, path, page, size, version);
CachedFileListPage cached = readCachedPage(cache, cacheKey);
if (cached != null) {
return cached.toPageResponse();
}
PageResponse<FileMetadataResponse> loaded = loader.get();
cache.put(cacheKey, CachedFileListPage.from(loaded));
return loaded;
}
@Override
public void touchDirectories(Long userId, Collection<String> paths) {
if (userId == null || paths == null || paths.isEmpty()) {
return;
}
Set<String> normalizedPaths = new LinkedHashSet<>();
for (String path : paths) {
String normalized = normalizeDirectoryPath(path);
if (normalized != null) {
normalizedPaths.add(normalized);
}
}
if (normalizedPaths.isEmpty()) {
return;
}
Duration ttl = Duration.ofSeconds(Math.max(
redisProperties.getCache().getDirectoryVersionTtlSeconds(),
redisProperties.getCache().getFilesListTtlSeconds() * 2
));
for (String path : normalizedPaths) {
String key = buildDirectoryVersionKey(userId, path);
stringRedisTemplate.opsForValue().increment(key);
stringRedisTemplate.expire(key, ttl);
}
}
private CachedFileListPage readCachedPage(Cache cache, String cacheKey) {
Cache.ValueWrapper wrapper = cache.get(cacheKey);
if (wrapper == null || wrapper.get() == null) {
return null;
}
Object cachedValue = wrapper.get();
if (cachedValue instanceof CachedFileListPage cachedFileListPage) {
return cachedFileListPage;
}
return objectMapper.convertValue(cachedValue, CachedFileListPage.class);
}
private long readDirectoryVersion(Long userId, String path) {
String value = stringRedisTemplate.opsForValue().get(buildDirectoryVersionKey(userId, path));
if (!StringUtils.hasText(value)) {
return 0L;
}
try {
return Long.parseLong(value.trim());
} catch (NumberFormatException ex) {
return 0L;
}
}
private String buildCacheKey(Long userId, String path, int page, int size, long version) {
return "u:" + userId
+ ":path:" + encode(path)
+ ":page:" + page
+ ":size:" + size
+ ":sort:" + SORT_CONTEXT
+ ":v:" + version;
}
private String buildDirectoryVersionKey(Long userId, String path) {
return redisProperties.getKeyPrefix()
+ ":" + redisProperties.getNamespaces().getCache()
+ ":files-list:version:u:" + userId
+ ":path:" + encode(path);
}
private String encode(String value) {
return Base64.getUrlEncoder().withoutPadding()
.encodeToString(value.getBytes(StandardCharsets.UTF_8));
}
private String normalizeDirectoryPath(String path) {
if (!StringUtils.hasText(path)) {
return "/";
}
String normalized = path.trim().replace("\\", "/");
while (normalized.contains("//")) {
normalized = normalized.replace("//", "/");
}
if (!normalized.startsWith("/")) {
normalized = "/" + normalized;
}
while (normalized.length() > 1 && normalized.endsWith("/")) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
private record CachedFileListPage(List<FileMetadataResponse> items, long total, int page, int size) {
private static CachedFileListPage from(PageResponse<FileMetadataResponse> response) {
return new CachedFileListPage(response.items(), response.total(), response.page(), response.size());
}
private PageResponse<FileMetadataResponse> toPageResponse() {
return new PageResponse<>(items, total, page, size);
}
}
}

View File

@@ -4,8 +4,23 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query; import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param; import org.springframework.data.repository.query.Param;
import java.util.Collection;
import java.util.List;
public interface StoredFileEntityRepository extends JpaRepository<StoredFileEntity, Long> { public interface StoredFileEntityRepository extends JpaRepository<StoredFileEntity, Long> {
interface FileEntityLinkStatsProjection {
Long getFileEntityId();
Long getLinkedStoredFileCount();
Long getLinkedOwnerCount();
String getSampleOwnerUsername();
String getSampleOwnerEmail();
}
@Query(""" @Query("""
select count(distinct relation.storedFile.id) select count(distinct relation.storedFile.id)
from StoredFileEntity relation from StoredFileEntity relation
@@ -41,4 +56,18 @@ public interface StoredFileEntityRepository extends JpaRepository<StoredFileEnti
where relation.fileEntity.id = :fileEntityId where relation.fileEntity.id = :fileEntityId
""") """)
String findSampleOwnerEmailByFileEntityId(@Param("fileEntityId") Long fileEntityId); String findSampleOwnerEmailByFileEntityId(@Param("fileEntityId") Long fileEntityId);
@Query("""
select relation.fileEntity.id as fileEntityId,
count(distinct relation.storedFile.id) as linkedStoredFileCount,
count(distinct owner.id) as linkedOwnerCount,
min(owner.username) as sampleOwnerUsername,
min(owner.email) as sampleOwnerEmail
from StoredFileEntity relation
join relation.storedFile storedFile
join storedFile.user owner
where relation.fileEntity.id in :fileEntityIds
group by relation.fileEntity.id
""")
List<FileEntityLinkStatsProjection> findAdminLinkStatsByFileEntityIds(@Param("fileEntityIds") Collection<Long> fileEntityIds);
} }

View File

@@ -8,11 +8,18 @@ import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param; import org.springframework.data.repository.query.Param;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
public interface StoredFileRepository extends JpaRepository<StoredFile, Long> { public interface StoredFileRepository extends JpaRepository<StoredFile, Long> {
interface UserStorageUsageProjection {
Long getUserId();
Long getUsedStorageBytes();
}
@EntityGraph(attributePaths = {"user", "blob"}) @EntityGraph(attributePaths = {"user", "blob"})
Page<StoredFile> findAllByOrderByCreatedAtDesc(Pageable pageable); Page<StoredFile> findAllByOrderByCreatedAtDesc(Pageable pageable);
@@ -104,6 +111,14 @@ public interface StoredFileRepository extends JpaRepository<StoredFile, Long> {
""") """)
long sumFileSizeByUserId(@Param("userId") Long userId); long sumFileSizeByUserId(@Param("userId") Long userId);
@Query("""
select f.user.id as userId, coalesce(sum(f.size), 0) as usedStorageBytes
from StoredFile f
where f.user.id in :userIds and f.directory = false and f.deletedAt is null
group by f.user.id
""")
List<UserStorageUsageProjection> sumFileSizeByUserIds(@Param("userIds") Collection<Long> userIds);
@Query(""" @Query("""
select coalesce(sum(f.size), 0) select coalesce(sum(f.size), 0)
from StoredFile f from StoredFile f

View File

@@ -0,0 +1,147 @@
package com.yoyuzh.files.core;
import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.storage.FileContentStorage;
import org.springframework.util.StringUtils;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
public final class WorkspaceNodeRulesService {
private final StoredFileRepository storedFileRepository;
private final FileContentStorage fileContentStorage;
public WorkspaceNodeRulesService(StoredFileRepository storedFileRepository,
FileContentStorage fileContentStorage) {
this.storedFileRepository = storedFileRepository;
this.fileContentStorage = fileContentStorage;
}
public String normalizeDirectoryPath(String path) {
if (!StringUtils.hasText(path) || "/".equals(path.trim())) {
return "/";
}
String normalized = path.replace("\\", "/").trim();
if (!normalized.startsWith("/")) {
normalized = "/" + normalized;
}
normalized = normalized.replaceAll("/{2,}", "/");
if (normalized.contains("..")) {
throw new BusinessException(ErrorCode.UNKNOWN, "路径不合法");
}
if (normalized.endsWith("/") && normalized.length() > 1) {
normalized = normalized.substring(0, normalized.length() - 1);
}
return normalized;
}
public String extractParentPath(String normalizedPath) {
int lastSlash = normalizedPath.lastIndexOf('/');
return lastSlash <= 0 ? "/" : normalizedPath.substring(0, lastSlash);
}
public String extractLeafName(String normalizedPath) {
return normalizedPath.substring(normalizedPath.lastIndexOf('/') + 1);
}
public String buildTargetLogicalPath(String normalizedTargetPath, String filename) {
return "/".equals(normalizedTargetPath)
? "/" + filename
: normalizedTargetPath + "/" + filename;
}
public String normalizeUploadFilename(String originalFilename) {
String filename = StringUtils.cleanPath(originalFilename);
if (!StringUtils.hasText(filename)) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件名不能为空");
}
return normalizeLeafName(filename);
}
public String normalizeLeafName(String filename) {
String cleaned = StringUtils.cleanPath(filename == null ? "" : filename).trim();
if (!StringUtils.hasText(cleaned)) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件名不能为空");
}
if (cleaned.contains("/") || cleaned.contains("\\") || cleaned.contains("..")) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件名不合法");
}
return cleaned;
}
public boolean existsNodeName(Long userId, String path, String filename) {
return storedFileRepository.existsByUserIdAndPathAndFilename(userId, path, filename);
}
public void ensureNodeNameAvailable(Long userId, String path, String filename, String errorMessage) {
if (existsNodeName(userId, path, filename)) {
throw new BusinessException(ErrorCode.UNKNOWN, errorMessage);
}
}
public void ensureDirectoryHierarchy(User user, String normalizedPath) {
if ("/".equals(normalizedPath)) {
return;
}
String[] segments = normalizedPath.substring(1).split("/");
String currentPath = "/";
for (String segment : segments) {
Optional<StoredFile> existing = storedFileRepository.findByUserIdAndPathAndFilename(user.getId(), currentPath, segment);
if (existing.isPresent()) {
if (!existing.get().isDirectory()) {
throw new BusinessException(ErrorCode.UNKNOWN, "目标路径不是目录");
}
currentPath = "/".equals(currentPath) ? "/" + segment : currentPath + "/" + segment;
continue;
}
String logicalPath = "/".equals(currentPath) ? "/" + segment : currentPath + "/" + segment;
fileContentStorage.ensureDirectory(user.getId(), logicalPath);
StoredFile storedFile = new StoredFile();
storedFile.setUser(user);
storedFile.setFilename(segment);
storedFile.setPath(currentPath);
storedFile.setContentType("directory");
storedFile.setSize(0L);
storedFile.setDirectory(true);
storedFileRepository.save(storedFile);
currentPath = logicalPath;
}
}
public void ensureExistingDirectoryPath(Long userId, String normalizedPath) {
if ("/".equals(normalizedPath)) {
return;
}
String[] segments = normalizedPath.substring(1).split("/");
String currentPath = "/";
for (String segment : segments) {
StoredFile directory = storedFileRepository.findByUserIdAndPathAndFilename(userId, currentPath, segment)
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "目标目录不存在"));
if (!directory.isDirectory()) {
throw new BusinessException(ErrorCode.UNKNOWN, "目标路径不是目录");
}
currentPath = "/".equals(currentPath) ? "/" + segment : currentPath + "/" + segment;
}
}
public void validateRecycleRestoreTargets(Long userId,
List<StoredFile> recycleGroupItems,
Function<StoredFile, String> recycleOriginalPathResolver) {
for (StoredFile item : recycleGroupItems) {
String originalPath = recycleOriginalPathResolver.apply(item);
if (existsNodeName(userId, originalPath, item.getFilename())) {
throw new BusinessException(ErrorCode.UNKNOWN, "原目录已存在同名文件,无法恢复");
}
}
}
}

View File

@@ -0,0 +1,6 @@
package com.yoyuzh.files.events;
public interface FileEventCrossInstancePublisher {
void publish(FileEvent event);
}

View File

@@ -0,0 +1,141 @@
package com.yoyuzh.files.events;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.io.IOException;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@Component
public class FileEventDispatcher {
private static final String READY_EVENT_NAME = "READY";
private final FileEventPayloadCodec payloadCodec;
private final ConcurrentHashMap<Long, Set<Subscription>> subscriptions = new ConcurrentHashMap<>();
public FileEventDispatcher(FileEventPayloadCodec payloadCodec) {
this.payloadCodec = payloadCodec;
}
public SseEmitter openStream(Long userId, String path, String clientId) {
String normalizedPath = normalizePath(path);
String normalizedClientId = normalizeClientId(clientId);
SseEmitter emitter = createEmitter();
Subscription subscription = new Subscription(emitter, normalizedPath, normalizedClientId);
subscriptions.computeIfAbsent(userId, ignored -> ConcurrentHashMap.newKeySet()).add(subscription);
emitter.onCompletion(() -> removeSubscription(userId, subscription));
emitter.onTimeout(() -> removeSubscription(userId, subscription));
emitter.onError(ex -> removeSubscription(userId, subscription));
try {
emitter.send(SseEmitter.event()
.name(READY_EVENT_NAME)
.data(payloadCodec.createReadyPayload(normalizedPath, normalizedClientId)));
} catch (IOException ex) {
removeSubscription(userId, subscription);
throw new IllegalStateException("Failed to initialize file event stream", ex);
}
return emitter;
}
public void broadcast(FileEvent event) {
Set<Subscription> userSubscriptions = subscriptions.get(event.getUserId());
if (userSubscriptions == null || userSubscriptions.isEmpty()) {
return;
}
for (Subscription subscription : userSubscriptions.toArray(new Subscription[0])) {
if (!subscription.matches(event)) {
continue;
}
try {
subscription.emitter.send(SseEmitter.event()
.name(event.getEventType().name())
.data(payloadCodec.createEmitterPayload(event)));
} catch (IOException | IllegalStateException ex) {
removeSubscription(event.getUserId(), subscription);
}
}
}
protected SseEmitter createEmitter() {
return new SseEmitter();
}
private void removeSubscription(Long userId, Subscription subscription) {
Set<Subscription> userSubscriptions = subscriptions.get(userId);
if (userSubscriptions == null) {
return;
}
userSubscriptions.remove(subscription);
if (userSubscriptions.isEmpty()) {
subscriptions.remove(userId, userSubscriptions);
}
}
private String normalizeClientId(String clientId) {
if (!StringUtils.hasText(clientId)) {
return null;
}
String cleaned = clientId.trim();
return cleaned.isEmpty() ? null : cleaned;
}
private String normalizePath(String path) {
if (!StringUtils.hasText(path)) {
return "/";
}
String cleaned = path.trim().replace("\\", "/");
while (cleaned.contains("//")) {
cleaned = cleaned.replace("//", "/");
}
if (!cleaned.startsWith("/")) {
cleaned = "/" + cleaned;
}
if (cleaned.length() > 1 && cleaned.endsWith("/")) {
cleaned = cleaned.substring(0, cleaned.length() - 1);
}
return cleaned;
}
private boolean isPathMatch(String filterPath, String eventPath) {
if (!StringUtils.hasText(filterPath) || "/".equals(filterPath)) {
return true;
}
if (!StringUtils.hasText(eventPath)) {
return false;
}
return Objects.equals(filterPath, eventPath) || eventPath.startsWith(filterPath + "/");
}
private final class Subscription {
private final SseEmitter emitter;
private final String path;
private final String clientId;
private Subscription(SseEmitter emitter, String path, String clientId) {
this.emitter = emitter;
this.path = path;
this.clientId = clientId;
}
private boolean matches(FileEvent event) {
boolean pathMatches;
if (event.getFromPath() != null && event.getToPath() != null) {
pathMatches = isPathMatch(path, event.getFromPath()) || isPathMatch(path, event.getToPath());
} else {
String eventPath = event.getToPath() != null ? event.getToPath() : event.getFromPath();
pathMatches = isPathMatch(path, eventPath);
}
if (!pathMatches) {
return false;
}
return clientId == null || event.getClientId() == null || !clientId.equals(event.getClientId());
}
}
}

View File

@@ -0,0 +1,23 @@
package com.yoyuzh.files.events;
import org.springframework.stereotype.Component;
import java.util.UUID;
@Component
public class FileEventInstanceIdentity {
private final String instanceId;
public FileEventInstanceIdentity() {
this(UUID.randomUUID().toString());
}
FileEventInstanceIdentity(String instanceId) {
this.instanceId = instanceId;
}
public String getInstanceId() {
return instanceId;
}
}

View File

@@ -0,0 +1,52 @@
package com.yoyuzh.files.events;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.Map;
@Component
public class FileEventPayloadCodec {
private final ObjectMapper objectMapper;
public FileEventPayloadCodec(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
public String toJson(Map<String, Object> payload) {
Map<String, Object> safePayload = payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload);
if (!safePayload.containsKey("createdAt")) {
safePayload.put("createdAt", LocalDateTime.now());
}
try {
return objectMapper.writeValueAsString(safePayload);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to serialize file event payload", ex);
}
}
public Map<String, Object> createReadyPayload(String path, String clientId) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventType", "READY");
payload.put("path", path);
payload.put("clientId", clientId);
payload.put("createdAt", LocalDateTime.now());
return payload;
}
public Map<String, Object> createEmitterPayload(FileEvent event) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventType", event.getEventType().name());
payload.put("fileId", event.getFileId());
payload.put("fromPath", event.getFromPath());
payload.put("toPath", event.getToPath());
payload.put("clientId", event.getClientId());
payload.put("createdAt", event.getCreatedAt());
payload.put("payload", event.getPayloadJson());
return payload;
}
}

View File

@@ -0,0 +1,17 @@
package com.yoyuzh.files.events;
import java.time.LocalDateTime;
record FileEventPubSubMessage(
String originInstanceId,
Long eventId,
Long userId,
FileEventType eventType,
Long fileId,
String fromPath,
String toPath,
String clientId,
String payloadJson,
LocalDateTime createdAt
) {
}

View File

@@ -0,0 +1,23 @@
package com.yoyuzh.files.events;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.listener.ChannelTopic;
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
@Configuration
public class FileEventRedisPubSubConfiguration {
@Bean
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public RedisMessageListenerContainer fileEventRedisMessageListenerContainer(
RedisConnectionFactory redisConnectionFactory,
RedisFileEventPubSubListener listener) {
RedisMessageListenerContainer container = new RedisMessageListenerContainer();
container.setConnectionFactory(redisConnectionFactory);
container.addMessageListener(listener, new ChannelTopic(listener.buildTopic()));
return container;
}
}

View File

@@ -1,7 +1,5 @@
package com.yoyuzh.files.events; package com.yoyuzh.files.events;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletRequest;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -12,46 +10,29 @@ import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes; import org.springframework.web.context.request.ServletRequestAttributes;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.io.IOException;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@Service @Service
public class FileEventService { public class FileEventService {
private static final String CLIENT_ID_HEADER = "X-Yoyuzh-Client-Id"; private static final String CLIENT_ID_HEADER = "X-Yoyuzh-Client-Id";
private static final String READY_EVENT_NAME = "READY";
private final FileEventRepository fileEventRepository; private final FileEventRepository fileEventRepository;
private final ObjectMapper objectMapper; private final FileEventPayloadCodec payloadCodec;
private final ConcurrentHashMap<Long, Set<Subscription>> subscriptions = new ConcurrentHashMap<>(); private final FileEventDispatcher fileEventDispatcher;
private final FileEventCrossInstancePublisher fileEventCrossInstancePublisher;
public FileEventService(FileEventRepository fileEventRepository, ObjectMapper objectMapper) { public FileEventService(FileEventRepository fileEventRepository,
FileEventPayloadCodec payloadCodec,
FileEventDispatcher fileEventDispatcher,
FileEventCrossInstancePublisher fileEventCrossInstancePublisher) {
this.fileEventRepository = fileEventRepository; this.fileEventRepository = fileEventRepository;
this.objectMapper = objectMapper; this.payloadCodec = payloadCodec;
this.fileEventDispatcher = fileEventDispatcher;
this.fileEventCrossInstancePublisher = fileEventCrossInstancePublisher;
} }
public SseEmitter openStream(User user, String path, String clientId) { public SseEmitter openStream(User user, String path, String clientId) {
String normalizedPath = normalizePath(path); return fileEventDispatcher.openStream(user.getId(), path, resolveClientId(clientId));
SseEmitter emitter = createEmitter();
Subscription subscription = new Subscription(emitter, normalizedPath, normalizeClientId(clientId));
subscriptions.computeIfAbsent(user.getId(), ignored -> ConcurrentHashMap.newKeySet()).add(subscription);
emitter.onCompletion(() -> removeSubscription(user.getId(), subscription));
emitter.onTimeout(() -> removeSubscription(user.getId(), subscription));
emitter.onError(ex -> removeSubscription(user.getId(), subscription));
try {
emitter.send(SseEmitter.event()
.name(READY_EVENT_NAME)
.data(createReadyPayload(normalizedPath, subscription.clientId)));
} catch (IOException ex) {
removeSubscription(user.getId(), subscription);
throw new IllegalStateException("Failed to initialize file event stream", ex);
}
return emitter;
} }
public FileEvent record(User user, public FileEvent record(User user,
@@ -68,7 +49,7 @@ public class FileEventService {
event.setFromPath(fromPath); event.setFromPath(fromPath);
event.setToPath(toPath); event.setToPath(toPath);
event.setClientId(resolveClientId(clientId)); event.setClientId(resolveClientId(clientId));
event.setPayloadJson(toJson(payload)); event.setPayloadJson(payloadCodec.toJson(payload));
fileEventRepository.save(event); fileEventRepository.save(event);
broadcast(event); broadcast(event);
return event; return event;
@@ -83,37 +64,14 @@ public class FileEventService {
return record(user, eventType, fileId, fromPath, toPath, null, payload); return record(user, eventType, fileId, fromPath, toPath, null, payload);
} }
protected SseEmitter createEmitter() { void broadcastReplicatedEvent(FileEvent event) {
return new SseEmitter(); fileEventDispatcher.broadcast(event);
} }
private void broadcast(FileEvent event) { private void broadcast(FileEvent event) {
Runnable broadcastTask = () -> { Runnable broadcastTask = () -> {
Set<Subscription> userSubscriptions = subscriptions.get(event.getUserId()); fileEventDispatcher.broadcast(event);
if (userSubscriptions == null || userSubscriptions.isEmpty()) { fileEventCrossInstancePublisher.publish(event);
return;
}
for (Subscription subscription : userSubscriptions.toArray(new Subscription[0])) {
if (!subscription.matches(event)) {
continue;
}
try {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventType", event.getEventType().name());
payload.put("fileId", event.getFileId());
payload.put("fromPath", event.getFromPath());
payload.put("toPath", event.getToPath());
payload.put("clientId", event.getClientId());
payload.put("createdAt", event.getCreatedAt());
payload.put("payload", event.getPayloadJson());
subscription.emitter.send(SseEmitter.event()
.name(event.getEventType().name())
.data(payload));
} catch (IOException | IllegalStateException ex) {
removeSubscription(event.getUserId(), subscription);
}
}
}; };
if (TransactionSynchronizationManager.isActualTransactionActive()) { if (TransactionSynchronizationManager.isActualTransactionActive()) {
@@ -129,32 +87,9 @@ public class FileEventService {
broadcastTask.run(); broadcastTask.run();
} }
private void removeSubscription(Long userId, Subscription subscription) {
Set<Subscription> userSubscriptions = subscriptions.get(userId);
if (userSubscriptions == null) {
return;
}
userSubscriptions.remove(subscription);
if (userSubscriptions.isEmpty()) {
subscriptions.remove(userId, userSubscriptions);
}
}
private String toJson(Map<String, Object> payload) {
Map<String, Object> safePayload = payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload);
if (!safePayload.containsKey("createdAt")) {
safePayload.put("createdAt", LocalDateTime.now());
}
try {
return objectMapper.writeValueAsString(safePayload);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to serialize file event payload", ex);
}
}
private String resolveClientId(String explicitClientId) { private String resolveClientId(String explicitClientId) {
if (StringUtils.hasText(explicitClientId)) { if (StringUtils.hasText(explicitClientId)) {
return normalizeClientId(explicitClientId); return explicitClientId.trim();
} }
ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
@@ -162,79 +97,7 @@ public class FileEventService {
return null; return null;
} }
HttpServletRequest request = attributes.getRequest(); HttpServletRequest request = attributes.getRequest();
return normalizeClientId(request.getHeader(CLIENT_ID_HEADER)); String requestClientId = request.getHeader(CLIENT_ID_HEADER);
} return StringUtils.hasText(requestClientId) ? requestClientId.trim() : null;
private String normalizeClientId(String clientId) {
if (!StringUtils.hasText(clientId)) {
return null;
}
String cleaned = clientId.trim();
return cleaned.isEmpty() ? null : cleaned;
}
private String normalizePath(String path) {
if (!StringUtils.hasText(path)) {
return "/";
}
String cleaned = path.trim().replace("\\", "/");
while (cleaned.contains("//")) {
cleaned = cleaned.replace("//", "/");
}
if (!cleaned.startsWith("/")) {
cleaned = "/" + cleaned;
}
if (cleaned.length() > 1 && cleaned.endsWith("/")) {
cleaned = cleaned.substring(0, cleaned.length() - 1);
}
return cleaned;
}
private boolean isPathMatch(String filterPath, String eventPath) {
if (!StringUtils.hasText(filterPath) || "/".equals(filterPath)) {
return true;
}
if (!StringUtils.hasText(eventPath)) {
return false;
}
return Objects.equals(filterPath, eventPath) || eventPath.startsWith(filterPath + "/");
}
private Map<String, Object> createReadyPayload(String path, String clientId) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventType", READY_EVENT_NAME);
payload.put("path", path);
payload.put("clientId", clientId);
payload.put("createdAt", LocalDateTime.now());
return payload;
}
private final class Subscription {
private final SseEmitter emitter;
private final String path;
private final String clientId;
private Subscription(SseEmitter emitter, String path, String clientId) {
this.emitter = emitter;
this.path = path;
this.clientId = clientId;
}
private boolean matches(FileEvent event) {
boolean pathMatches;
if (event.getFromPath() != null && event.getToPath() != null) {
pathMatches = FileEventService.this.isPathMatch(path, event.getFromPath())
|| FileEventService.this.isPathMatch(path, event.getToPath());
} else {
String eventPath = event.getToPath() != null ? event.getToPath() : event.getFromPath();
pathMatches = FileEventService.this.isPathMatch(path, eventPath);
}
if (!pathMatches) {
return false;
}
return clientId == null || event.getClientId() == null || !clientId.equals(event.getClientId());
}
} }
} }

View File

@@ -0,0 +1,14 @@
package com.yoyuzh.files.events;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "false", matchIfMissing = true)
public class NoOpFileEventCrossInstancePublisher implements FileEventCrossInstancePublisher {
@Override
public void publish(FileEvent event) {
// Redis disabled: keep single-instance in-memory broadcast behavior.
}
}

View File

@@ -0,0 +1,88 @@
package com.yoyuzh.files.events;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.config.AppRedisProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.Message;
import org.springframework.data.redis.connection.MessageListener;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.nio.charset.StandardCharsets;
@Component
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public class RedisFileEventPubSubListener implements MessageListener {
private final ObjectMapper objectMapper;
private final AppRedisProperties redisProperties;
private final FileEventService fileEventService;
private final String instanceId;
@Autowired
public RedisFileEventPubSubListener(ObjectMapper objectMapper,
AppRedisProperties redisProperties,
FileEventService fileEventService,
FileEventInstanceIdentity instanceIdentity) {
this(objectMapper, redisProperties, fileEventService, instanceIdentity.getInstanceId());
}
RedisFileEventPubSubListener(ObjectMapper objectMapper,
AppRedisProperties redisProperties,
FileEventService fileEventService,
String instanceId) {
this.objectMapper = objectMapper;
this.redisProperties = redisProperties;
this.fileEventService = fileEventService;
this.instanceId = instanceId;
}
@Override
public void onMessage(Message message, byte[] pattern) {
String payload = new String(message.getBody(), StandardCharsets.UTF_8);
if (!StringUtils.hasText(payload)) {
return;
}
FileEventPubSubMessage pubSubMessage;
try {
pubSubMessage = parsePayload(payload);
} catch (IllegalStateException ex) {
return;
}
if (instanceId.equals(pubSubMessage.originInstanceId())) {
return;
}
fileEventService.broadcastReplicatedEvent(toEvent(pubSubMessage));
}
String buildTopic() {
return redisProperties.getKeyPrefix()
+ ":"
+ redisProperties.getNamespaces().getFileEvents()
+ ":pubsub";
}
private FileEventPubSubMessage parsePayload(String payload) {
try {
return objectMapper.readValue(payload, FileEventPubSubMessage.class);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to parse file event pub/sub payload", ex);
}
}
private FileEvent toEvent(FileEventPubSubMessage message) {
FileEvent event = new FileEvent();
event.setId(message.eventId());
event.setUserId(message.userId());
event.setEventType(message.eventType());
event.setFileId(message.fileId());
event.setFromPath(message.fromPath());
event.setToPath(message.toPath());
event.setClientId(message.clientId());
event.setPayloadJson(message.payloadJson());
event.setCreatedAt(message.createdAt());
return event;
}
}

View File

@@ -0,0 +1,71 @@
package com.yoyuzh.files.events;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.config.AppRedisProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public class RedisFileEventPubSubPublisher implements FileEventCrossInstancePublisher {
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final AppRedisProperties redisProperties;
private final String instanceId;
@Autowired
public RedisFileEventPubSubPublisher(StringRedisTemplate stringRedisTemplate,
ObjectMapper objectMapper,
AppRedisProperties redisProperties,
FileEventInstanceIdentity instanceIdentity) {
this(stringRedisTemplate, objectMapper, redisProperties, instanceIdentity.getInstanceId());
}
RedisFileEventPubSubPublisher(StringRedisTemplate stringRedisTemplate,
ObjectMapper objectMapper,
AppRedisProperties redisProperties,
String instanceId) {
this.stringRedisTemplate = stringRedisTemplate;
this.objectMapper = objectMapper;
this.redisProperties = redisProperties;
this.instanceId = instanceId;
}
@Override
public void publish(FileEvent event) {
try {
stringRedisTemplate.convertAndSend(
buildTopic(),
objectMapper.writeValueAsString(toMessage(event))
);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to serialize file event pub/sub payload", ex);
}
}
String buildTopic() {
return redisProperties.getKeyPrefix()
+ ":"
+ redisProperties.getNamespaces().getFileEvents()
+ ":pubsub";
}
private FileEventPubSubMessage toMessage(FileEvent event) {
return new FileEventPubSubMessage(
instanceId,
event.getId(),
event.getUserId(),
event.getEventType(),
event.getFileId(),
event.getFromPath(),
event.getToPath(),
event.getClientId(),
event.getPayloadJson(),
event.getCreatedAt()
);
}
}

View File

@@ -1,8 +1,5 @@
package com.yoyuzh.files.tasks; package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository; import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.files.core.FileMetadataResponse; import com.yoyuzh.files.core.FileMetadataResponse;
@@ -23,16 +20,16 @@ public class ArchiveBackgroundTaskHandler implements BackgroundTaskHandler {
private final StoredFileRepository storedFileRepository; private final StoredFileRepository storedFileRepository;
private final UserRepository userRepository; private final UserRepository userRepository;
private final FileService fileService; private final FileService fileService;
private final ObjectMapper objectMapper; private final BackgroundTaskStateManager stateManager;
public ArchiveBackgroundTaskHandler(StoredFileRepository storedFileRepository, public ArchiveBackgroundTaskHandler(StoredFileRepository storedFileRepository,
UserRepository userRepository, UserRepository userRepository,
FileService fileService, FileService fileService,
ObjectMapper objectMapper) { BackgroundTaskStateManager stateManager) {
this.storedFileRepository = storedFileRepository; this.storedFileRepository = storedFileRepository;
this.userRepository = userRepository; this.userRepository = userRepository;
this.fileService = fileService; this.fileService = fileService;
this.objectMapper = objectMapper; this.stateManager = stateManager;
} }
@Override @Override
@@ -48,10 +45,14 @@ public class ArchiveBackgroundTaskHandler implements BackgroundTaskHandler {
@Override @Override
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) { public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
Map<String, Object> state = parseState(task.getPrivateStateJson(), task.getPublicStateJson()); Map<String, Object> state = stateManager.mergeJsonObjects(
Long fileId = extractLong(state.get("fileId")); task.getPublicStateJson(),
String outputPath = extractText(state.get("outputPath")); task.getPrivateStateJson(),
String outputFilename = extractText(state.get("outputFilename")); "archive task state is invalid"
);
Long fileId = stateManager.readLong(state.get("fileId"));
String outputPath = stateManager.readText(state.get("outputPath"));
String outputFilename = stateManager.readText(state.get("outputFilename"));
if (fileId == null) { if (fileId == null) {
throw new IllegalStateException("archive task missing fileId"); throw new IllegalStateException("archive task missing fileId");
} }
@@ -127,38 +128,4 @@ public class ArchiveBackgroundTaskHandler implements BackgroundTaskHandler {
return Math.min(100, (int) Math.floor((processed * 100.0d) / total)); return Math.min(100, (int) Math.floor((processed * 100.0d) / total));
} }
private Map<String, Object> parseState(String privateStateJson, String publicStateJson) {
Map<String, Object> state = new LinkedHashMap<>(parseJsonObject(publicStateJson));
state.putAll(parseJsonObject(privateStateJson));
return state;
}
private Map<String, Object> parseJsonObject(String json) {
if (!StringUtils.hasText(json)) {
return Map.of();
}
try {
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("archive task state is invalid", ex);
}
}
private Long extractLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text && StringUtils.hasText(text)) {
return Long.parseLong(text.trim());
}
return null;
}
private String extractText(Object value) {
if (value instanceof String text && StringUtils.hasText(text)) {
return text.trim();
}
return null;
}
} }

View File

@@ -11,6 +11,7 @@ import jakarta.persistence.Index;
import jakarta.persistence.PrePersist; import jakarta.persistence.PrePersist;
import jakarta.persistence.PreUpdate; import jakarta.persistence.PreUpdate;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import jakarta.persistence.UniqueConstraint;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@@ -18,8 +19,9 @@ import java.time.LocalDateTime;
@Table(name = "portal_background_task", indexes = { @Table(name = "portal_background_task", indexes = {
@Index(name = "idx_background_task_user_created_at", columnList = "user_id,created_at"), @Index(name = "idx_background_task_user_created_at", columnList = "user_id,created_at"),
@Index(name = "idx_background_task_status_created_at", columnList = "status,created_at"), @Index(name = "idx_background_task_status_created_at", columnList = "status,created_at"),
@Index(name = "idx_background_task_status_lease_expires_at", columnList = "status,lease_expires_at"), @Index(name = "idx_background_task_status_lease_expires_at", columnList = "status,lease_expires_at")
@Index(name = "idx_background_task_correlation_id", columnList = "correlation_id") }, uniqueConstraints = {
@UniqueConstraint(name = "uk_background_task_correlation_id", columnNames = "correlation_id")
}) })
public class BackgroundTask { public class BackgroundTask {

View File

@@ -0,0 +1,55 @@
package com.yoyuzh.files.tasks;
import com.yoyuzh.auth.User;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import java.util.Map;
import java.util.Optional;
@Service
@RequiredArgsConstructor
public class BackgroundTaskCommandService {
private final BackgroundTaskService backgroundTaskService;
public BackgroundTask createQueuedFileTask(User user,
BackgroundTaskType type,
Long fileId,
String requestedPath,
String correlationId) {
return backgroundTaskService.createQueuedFileTask(user, type, fileId, requestedPath, correlationId);
}
public Optional<BackgroundTask> createQueuedAutoMediaMetadataTask(Long userId,
Long fileId,
String correlationId) {
return backgroundTaskService.createQueuedAutoMediaMetadataTask(userId, fileId, correlationId);
}
public BackgroundTask createQueuedTask(User user,
BackgroundTaskType type,
Map<String, Object> publicState,
Map<String, Object> privateState,
String correlationId) {
return backgroundTaskService.createQueuedTask(user, type, publicState, privateState, correlationId);
}
public Page<BackgroundTask> listOwnedTasks(User user, Pageable pageable) {
return backgroundTaskService.listOwnedTasks(user, pageable);
}
public BackgroundTask getOwnedTask(User user, Long id) {
return backgroundTaskService.getOwnedTask(user, id);
}
public BackgroundTask cancelOwnedTask(User user, Long id) {
return backgroundTaskService.cancelOwnedTask(user, id);
}
public BackgroundTask retryOwnedTask(User user, Long id) {
return backgroundTaskService.retryOwnedTask(user, id);
}
}

View File

@@ -0,0 +1,243 @@
package com.yoyuzh.files.tasks;
import com.yoyuzh.api.v2.ApiV2ErrorCode;
import com.yoyuzh.api.v2.ApiV2Exception;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@Service
@RequiredArgsConstructor
public class BackgroundTaskExecutionService {
private static final List<String> RETRY_TRANSIENT_STATE_KEYS = List.of(
BackgroundTaskStateKeys.RETRY_SCHEDULED,
BackgroundTaskStateKeys.NEXT_RETRY_AT,
BackgroundTaskStateKeys.RETRY_DELAY_SECONDS,
BackgroundTaskStateKeys.LAST_FAILURE_MESSAGE,
BackgroundTaskStateKeys.LAST_FAILURE_AT,
BackgroundTaskStateKeys.FAILURE_CATEGORY
);
private static final List<String> RUNNING_TRANSIENT_STATE_KEYS = List.of(
BackgroundTaskStateKeys.WORKER_OWNER,
BackgroundTaskStateKeys.LEASE_EXPIRES_AT
);
private static final int EXPIRED_RUNNING_TASK_BATCH_SIZE = 100;
private final BackgroundTaskRepository backgroundTaskRepository;
private final BackgroundTaskRetryPolicy retryPolicy;
private final BackgroundTaskStateManager stateManager;
@Transactional
public int requeueExpiredRunningTasks() {
LocalDateTime now = LocalDateTime.now();
int recovered = 0;
for (Long taskId : backgroundTaskRepository.findExpiredRunningTaskIds(
BackgroundTaskStatus.RUNNING,
now,
PageRequest.of(0, EXPIRED_RUNNING_TASK_BATCH_SIZE)
)) {
int requeued = backgroundTaskRepository.requeueExpiredRunningTask(
taskId,
BackgroundTaskStatus.RUNNING,
BackgroundTaskStatus.QUEUED,
now,
now
);
if (requeued != 1) {
continue;
}
BackgroundTask task = backgroundTaskRepository.findById(taskId)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
resetTaskToQueued(task);
backgroundTaskRepository.save(task);
recovered += 1;
}
return recovered;
}
public List<Long> findQueuedTaskIds(int limit) {
if (limit <= 0) {
return List.of();
}
return backgroundTaskRepository.findReadyTaskIdsByStatusOrder(
BackgroundTaskStatus.QUEUED,
LocalDateTime.now(),
PageRequest.of(0, limit)
);
}
@Transactional
public Optional<BackgroundTask> claimQueuedTask(Long id, String workerOwner, long leaseDurationSeconds) {
LocalDateTime now = LocalDateTime.now();
LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds));
int claimed = backgroundTaskRepository.claimQueuedTask(
id,
BackgroundTaskStatus.QUEUED,
BackgroundTaskStatus.RUNNING,
workerOwner,
leaseExpiresAt,
now,
now
);
if (claimed != 1) {
return Optional.empty();
}
Optional<BackgroundTask> task = backgroundTaskRepository.findById(id);
task.ifPresent(claimedTask -> {
claimedTask.setLeaseOwner(workerOwner);
claimedTask.setLeaseExpiresAt(leaseExpiresAt);
claimedTask.setHeartbeatAt(now);
claimedTask.setPublicStateJson(stateManager.merge(
claimedTask.getPublicStateJson(),
stateManager.runningStatePatch(claimedTask, workerOwner, now, leaseExpiresAt, true),
RETRY_TRANSIENT_STATE_KEYS
));
});
task.ifPresent(backgroundTaskRepository::save);
return task;
}
@Transactional
public BackgroundTask markWorkerTaskProgress(Long id,
String workerOwner,
Map<String, Object> publicStatePatch,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
task.setLeaseOwner(workerOwner);
task.setLeaseExpiresAt(leaseTouch.leaseExpiresAt());
task.setHeartbeatAt(leaseTouch.now());
Map<String, Object> nextPatch = new LinkedHashMap<>(stateManager.runningStatePatch(
task,
workerOwner,
leaseTouch.now(),
leaseTouch.leaseExpiresAt(),
false
));
if (publicStatePatch != null) {
nextPatch.putAll(publicStatePatch);
}
task.setPublicStateJson(stateManager.merge(task.getPublicStateJson(), nextPatch));
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markWorkerTaskCompleted(Long id,
String workerOwner,
Map<String, Object> publicStatePatch,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
task.setPublicStateJson(stateManager.merge(
task.getPublicStateJson(),
stateManager.completedStatePatch(task, leaseTouch.now(), publicStatePatch),
stateManager.removableKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
));
task.setStatus(BackgroundTaskStatus.COMPLETED);
task.setNextRunAt(null);
clearLease(task);
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markWorkerTaskFailed(Long id,
String workerOwner,
String errorMessage,
BackgroundTaskFailureCategory failureCategory,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
String normalizedErrorMessage = StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed";
LocalDateTime now = leaseTouch.now();
if (failureCategory.isRetryable() && retryPolicy.hasRemainingAttempts(task)) {
long retryDelaySeconds = retryPolicy.resolveRetryDelaySeconds(task.getType(), failureCategory, task.getAttemptCount());
LocalDateTime nextRunAt = now.plusSeconds(retryDelaySeconds);
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setNextRunAt(nextRunAt);
clearLease(task);
task.setFinishedAt(null);
task.setErrorMessage(null);
task.setPublicStateJson(stateManager.merge(
task.getPublicStateJson(),
stateManager.retryQueuedStatePatch(
task,
normalizedErrorMessage,
failureCategory,
nextRunAt,
retryDelaySeconds,
now
),
RUNNING_TRANSIENT_STATE_KEYS
));
return backgroundTaskRepository.save(task);
}
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(stateManager.merge(
task.getPublicStateJson(),
stateManager.failedStatePatch(task, normalizedErrorMessage, failureCategory, now),
stateManager.removableKeys(
List.of(
BackgroundTaskStateKeys.RETRY_SCHEDULED,
BackgroundTaskStateKeys.NEXT_RETRY_AT,
BackgroundTaskStateKeys.RETRY_DELAY_SECONDS
),
RUNNING_TRANSIENT_STATE_KEYS
)
));
task.setStatus(BackgroundTaskStatus.FAILED);
task.setFinishedAt(now);
task.setErrorMessage(normalizedErrorMessage);
return backgroundTaskRepository.save(task);
}
private void resetTaskToQueued(BackgroundTask task) {
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(stateManager.resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts()));
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setFinishedAt(null);
task.setErrorMessage(null);
}
private LeaseTouch refreshLease(Long id, String workerOwner, long leaseDurationSeconds) {
LocalDateTime now = LocalDateTime.now();
LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds));
int refreshed = backgroundTaskRepository.refreshRunningTaskLease(
id,
BackgroundTaskStatus.RUNNING,
workerOwner,
leaseExpiresAt,
now,
now
);
if (refreshed != 1) {
throw new BackgroundTaskLeaseLostException(id, workerOwner);
}
return new LeaseTouch(now, leaseExpiresAt);
}
private void clearLease(BackgroundTask task) {
task.setLeaseOwner(null);
task.setLeaseExpiresAt(null);
task.setHeartbeatAt(null);
}
private record LeaseTouch(LocalDateTime now, LocalDateTime leaseExpiresAt) {
}
}

View File

@@ -0,0 +1,41 @@
package com.yoyuzh.files.tasks;
import org.springframework.stereotype.Component;
@Component
public class BackgroundTaskRetryPolicy {
public int resolveMaxAttempts(BackgroundTaskType type) {
return switch (type) {
case ARCHIVE -> 4;
case EXTRACT -> 3;
case MEDIA_META -> 2;
default -> 1;
};
}
public boolean hasRemainingAttempts(BackgroundTask task) {
return task.getAttemptCount() != null
&& task.getMaxAttempts() != null
&& task.getAttemptCount() < task.getMaxAttempts();
}
public long resolveRetryDelaySeconds(BackgroundTaskType type,
BackgroundTaskFailureCategory failureCategory,
Integer attemptCount) {
int safeAttemptCount = attemptCount == null ? 1 : Math.max(1, attemptCount);
long baseDelaySeconds = switch (type) {
case ARCHIVE -> 30L;
case EXTRACT -> 45L;
case MEDIA_META -> 15L;
default -> 30L;
};
if (failureCategory == BackgroundTaskFailureCategory.RATE_LIMITED) {
baseDelaySeconds *= 4L;
} else if (failureCategory == BackgroundTaskFailureCategory.UNKNOWN) {
baseDelaySeconds *= 2L;
}
long delay = baseDelaySeconds * (1L << Math.min(safeAttemptCount - 1, 2));
return Math.min(delay, baseDelaySeconds * 4L);
}
}

View File

@@ -1,21 +1,20 @@
package com.yoyuzh.files.tasks; package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.api.v2.ApiV2ErrorCode; import com.yoyuzh.api.v2.ApiV2ErrorCode;
import com.yoyuzh.api.v2.ApiV2Exception; import com.yoyuzh.api.v2.ApiV2Exception;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.common.lock.DistributedLockService;
import com.yoyuzh.files.core.StoredFile; import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
import jakarta.transaction.Transactional; import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor; import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import java.time.Duration;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
@@ -25,29 +24,23 @@ import java.util.Optional;
import java.util.UUID; import java.util.UUID;
@Service @Service
@RequiredArgsConstructor
public class BackgroundTaskService { public class BackgroundTaskService {
static final String STATE_PHASE_KEY = "phase"; static final String STATE_PHASE_KEY = BackgroundTaskStateKeys.PHASE;
static final String STATE_ATTEMPT_COUNT_KEY = "attemptCount"; static final String STATE_ATTEMPT_COUNT_KEY = BackgroundTaskStateKeys.ATTEMPT_COUNT;
static final String STATE_MAX_ATTEMPTS_KEY = "maxAttempts"; static final String STATE_MAX_ATTEMPTS_KEY = BackgroundTaskStateKeys.MAX_ATTEMPTS;
static final String STATE_RETRY_SCHEDULED_KEY = "retryScheduled"; static final String STATE_RETRY_SCHEDULED_KEY = BackgroundTaskStateKeys.RETRY_SCHEDULED;
static final String STATE_NEXT_RETRY_AT_KEY = "nextRetryAt"; static final String STATE_NEXT_RETRY_AT_KEY = BackgroundTaskStateKeys.NEXT_RETRY_AT;
static final String STATE_RETRY_DELAY_SECONDS_KEY = "retryDelaySeconds"; static final String STATE_RETRY_DELAY_SECONDS_KEY = BackgroundTaskStateKeys.RETRY_DELAY_SECONDS;
static final String STATE_LAST_FAILURE_MESSAGE_KEY = "lastFailureMessage"; static final String STATE_LAST_FAILURE_MESSAGE_KEY = BackgroundTaskStateKeys.LAST_FAILURE_MESSAGE;
static final String STATE_LAST_FAILURE_AT_KEY = "lastFailureAt"; static final String STATE_LAST_FAILURE_AT_KEY = BackgroundTaskStateKeys.LAST_FAILURE_AT;
static final String STATE_FAILURE_CATEGORY_KEY = "failureCategory"; static final String STATE_FAILURE_CATEGORY_KEY = BackgroundTaskStateKeys.FAILURE_CATEGORY;
static final String STATE_WORKER_OWNER_KEY = "workerOwner"; static final String STATE_WORKER_OWNER_KEY = BackgroundTaskStateKeys.WORKER_OWNER;
static final String STATE_HEARTBEAT_AT_KEY = "heartbeatAt"; static final String STATE_HEARTBEAT_AT_KEY = BackgroundTaskStateKeys.HEARTBEAT_AT;
static final String STATE_LEASE_EXPIRES_AT_KEY = "leaseExpiresAt"; static final String STATE_LEASE_EXPIRES_AT_KEY = BackgroundTaskStateKeys.LEASE_EXPIRES_AT;
static final String STATE_STARTED_AT_KEY = "startedAt"; static final String STATE_STARTED_AT_KEY = BackgroundTaskStateKeys.STARTED_AT;
private static final List<String> ZIP_COMPATIBLE_EXTENSIONS = List.of(".zip", ".jar", ".war"); private static final List<String> ZIP_COMPATIBLE_EXTENSIONS = List.of(".zip", ".jar", ".war");
private static final List<String> MEDIA_EXTENSIONS = List.of(
".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg",
".mp4", ".mov", ".mkv", ".webm", ".avi",
".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"
);
private static final List<String> RETRY_TRANSIENT_STATE_KEYS = List.of( private static final List<String> RETRY_TRANSIENT_STATE_KEYS = List.of(
STATE_RETRY_SCHEDULED_KEY, STATE_RETRY_SCHEDULED_KEY,
STATE_NEXT_RETRY_AT_KEY, STATE_NEXT_RETRY_AT_KEY,
@@ -60,11 +53,43 @@ public class BackgroundTaskService {
STATE_WORKER_OWNER_KEY, STATE_WORKER_OWNER_KEY,
STATE_LEASE_EXPIRES_AT_KEY STATE_LEASE_EXPIRES_AT_KEY
); );
private static final int EXPIRED_RUNNING_TASK_BATCH_SIZE = 100; private static final Duration CORRELATION_LOCK_TTL = Duration.ofSeconds(5);
private final BackgroundTaskRepository backgroundTaskRepository; private final BackgroundTaskRepository backgroundTaskRepository;
private final StoredFileRepository storedFileRepository; private final StoredFileRepository storedFileRepository;
private final ObjectMapper objectMapper; private final DistributedLockService distributedLockService;
private final BackgroundTaskRetryPolicy retryPolicy;
private final BackgroundTaskStateManager stateManager;
@Autowired
public BackgroundTaskService(BackgroundTaskRepository backgroundTaskRepository,
StoredFileRepository storedFileRepository,
com.fasterxml.jackson.databind.ObjectMapper objectMapper,
DistributedLockService distributedLockService,
BackgroundTaskRetryPolicy retryPolicy,
BackgroundTaskStateManager stateManager) {
this.backgroundTaskRepository = backgroundTaskRepository;
this.storedFileRepository = storedFileRepository;
this.distributedLockService = distributedLockService == null
? DistributedLockService.noOp()
: distributedLockService;
this.retryPolicy = retryPolicy == null ? new BackgroundTaskRetryPolicy() : retryPolicy;
this.stateManager = stateManager == null ? new BackgroundTaskStateManager(objectMapper) : stateManager;
}
BackgroundTaskService(BackgroundTaskRepository backgroundTaskRepository,
StoredFileRepository storedFileRepository,
com.fasterxml.jackson.databind.ObjectMapper objectMapper,
DistributedLockService distributedLockService) {
this(
backgroundTaskRepository,
storedFileRepository,
objectMapper,
distributedLockService,
new BackgroundTaskRetryPolicy(),
new BackgroundTaskStateManager(objectMapper)
);
}
@Transactional @Transactional
public BackgroundTask createQueuedFileTask(User user, public BackgroundTask createQueuedFileTask(User user,
@@ -78,27 +103,40 @@ public class BackgroundTaskService {
if (!logicalPath.equals(normalizeLogicalPath(requestedPath))) { if (!logicalPath.equals(normalizeLogicalPath(requestedPath))) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task path does not match file path"); throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "task path does not match file path");
} }
validateTaskTarget(type, file); return createQueuedFileTaskInternal(user.getId(), type, file, correlationId, false);
}
Map<String, Object> publicState = fileState(file, logicalPath);
Map<String, Object> privateState = new LinkedHashMap<>(publicState); @Transactional
privateState.put("taskType", type.name()); public Optional<BackgroundTask> createQueuedAutoMediaMetadataTask(Long userId,
if (type == BackgroundTaskType.ARCHIVE) { Long fileId,
String outputPath = file.getPath(); String correlationId) {
String outputFilename = file.getFilename() + ".zip"; String normalizedCorrelationId = StringUtils.hasText(correlationId)
publicState.put("outputPath", outputPath); ? correlationId.trim()
publicState.put("outputFilename", outputFilename); : "media-meta:auto:file:" + fileId;
privateState.put("outputPath", outputPath); try {
privateState.put("outputFilename", outputFilename); return distributedLockService.executeWithLock(
} else if (type == BackgroundTaskType.EXTRACT) { correlationLockName(normalizedCorrelationId),
String outputPath = file.getPath(); CORRELATION_LOCK_TTL,
String outputDirectoryName = deriveExtractOutputDirectoryName(file.getFilename()); () -> {
publicState.put("outputPath", outputPath); if (backgroundTaskRepository.existsByCorrelationId(normalizedCorrelationId)) {
publicState.put("outputDirectoryName", outputDirectoryName); return Optional.empty();
privateState.put("outputPath", outputPath); }
privateState.put("outputDirectoryName", outputDirectoryName);
return storedFileRepository.findByIdAndUserIdAndDeletedAtIsNull(fileId, userId)
.filter(file -> !file.isDirectory())
.filter(file -> MediaTaskSupport.isMediaLike(file.getFilename(), file.getContentType()))
.map(file -> createQueuedFileTaskInternal(
userId,
BackgroundTaskType.MEDIA_META,
file,
normalizedCorrelationId,
true
));
}
);
} catch (DataIntegrityViolationException ex) {
return Optional.empty();
} }
return createQueuedTask(user, type, publicState, privateState, correlationId);
} }
@Transactional @Transactional
@@ -107,20 +145,36 @@ public class BackgroundTaskService {
Map<String, Object> publicState, Map<String, Object> publicState,
Map<String, Object> privateState, Map<String, Object> privateState,
String correlationId) { String correlationId) {
return createQueuedTask(user.getId(), type, publicState, privateState, correlationId);
}
private BackgroundTask createQueuedTask(Long userId,
BackgroundTaskType type,
Map<String, Object> publicState,
Map<String, Object> privateState,
String correlationId) {
return createQueuedTask(userId, type, publicState, privateState, correlationId, false);
}
private BackgroundTask createQueuedTask(Long userId,
BackgroundTaskType type,
Map<String, Object> publicState,
Map<String, Object> privateState,
String correlationId,
boolean flushOnSave) {
BackgroundTask task = new BackgroundTask(); BackgroundTask task = new BackgroundTask();
task.setUserId(user.getId()); task.setUserId(userId);
task.setType(type); task.setType(type);
task.setStatus(BackgroundTaskStatus.QUEUED); task.setStatus(BackgroundTaskStatus.QUEUED);
task.setAttemptCount(0); task.setAttemptCount(0);
task.setMaxAttempts(resolveMaxAttempts(type)); task.setMaxAttempts(retryPolicy.resolveMaxAttempts(type));
task.setNextRunAt(null); task.setNextRunAt(null);
Map<String, Object> nextPublicState = new LinkedHashMap<>(publicState == null ? Map.of() : publicState); task.setPublicStateJson(stateManager.createInitialPublicState(publicState, task.getAttemptCount(), task.getMaxAttempts()));
nextPublicState.put(STATE_PHASE_KEY, "queued"); task.setPrivateStateJson(stateManager.toJson(privateState));
nextPublicState.putAll(retryStatePatch(task.getAttemptCount(), task.getMaxAttempts()));
task.setPublicStateJson(toJson(nextPublicState));
task.setPrivateStateJson(toJson(privateState));
task.setCorrelationId(normalizeCorrelationId(correlationId)); task.setCorrelationId(normalizeCorrelationId(correlationId));
return backgroundTaskRepository.save(task); return flushOnSave
? backgroundTaskRepository.saveAndFlush(task)
: backgroundTaskRepository.save(task);
} }
public Page<BackgroundTask> listOwnedTasks(User user, Pageable pageable) { public Page<BackgroundTask> listOwnedTasks(User user, Pageable pageable) {
@@ -143,15 +197,10 @@ public class BackgroundTaskService {
task.setStatus(BackgroundTaskStatus.CANCELLED); task.setStatus(BackgroundTaskStatus.CANCELLED);
task.setNextRunAt(null); task.setNextRunAt(null);
clearLease(task); clearLease(task);
task.setPublicStateJson(mergePublicStateJson( task.setPublicStateJson(stateManager.merge(
task.getPublicStateJson(), task.getPublicStateJson(),
Map.of( stateManager.cancelledStatePatch(task, LocalDateTime.now()),
STATE_PHASE_KEY, "cancelled", stateManager.removableKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
),
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
)); ));
task.setFinishedAt(LocalDateTime.now()); task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null); task.setErrorMessage(null);
@@ -171,7 +220,7 @@ public class BackgroundTaskService {
task.setAttemptCount(0); task.setAttemptCount(0);
task.setNextRunAt(null); task.setNextRunAt(null);
clearLease(task); clearLease(task);
task.setPublicStateJson(resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts())); task.setPublicStateJson(stateManager.resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts()));
task.setStatus(BackgroundTaskStatus.QUEUED); task.setStatus(BackgroundTaskStatus.QUEUED);
task.setFinishedAt(null); task.setFinishedAt(null);
task.setErrorMessage(null); task.setErrorMessage(null);
@@ -185,7 +234,7 @@ public class BackgroundTaskService {
return task; return task;
} }
task.setStatus(BackgroundTaskStatus.RUNNING); task.setStatus(BackgroundTaskStatus.RUNNING);
task.setPublicStateJson(mergePublicStateJson( task.setPublicStateJson(stateManager.merge(
task.getPublicStateJson(), task.getPublicStateJson(),
Map.of( Map.of(
STATE_PHASE_KEY, "running", STATE_PHASE_KEY, "running",
@@ -206,15 +255,10 @@ public class BackgroundTaskService {
task.setStatus(BackgroundTaskStatus.COMPLETED); task.setStatus(BackgroundTaskStatus.COMPLETED);
task.setNextRunAt(null); task.setNextRunAt(null);
clearLease(task); clearLease(task);
task.setPublicStateJson(mergePublicStateJson( task.setPublicStateJson(stateManager.merge(
task.getPublicStateJson(), task.getPublicStateJson(),
Map.of( stateManager.completedStatePatch(task, LocalDateTime.now(), null),
STATE_PHASE_KEY, "completed", stateManager.removableKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
),
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
)); ));
task.setFinishedAt(LocalDateTime.now()); task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null); task.setErrorMessage(null);
@@ -230,201 +274,18 @@ public class BackgroundTaskService {
task.setStatus(BackgroundTaskStatus.FAILED); task.setStatus(BackgroundTaskStatus.FAILED);
task.setNextRunAt(null); task.setNextRunAt(null);
clearLease(task); clearLease(task);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
Map.of(
STATE_PHASE_KEY, "failed",
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_LAST_FAILURE_MESSAGE_KEY, StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed",
STATE_LAST_FAILURE_AT_KEY, LocalDateTime.now().toString(),
STATE_FAILURE_CATEGORY_KEY, BackgroundTaskFailureCategory.UNKNOWN.name(),
STATE_HEARTBEAT_AT_KEY, LocalDateTime.now().toString()
),
removableStateKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY), RUNNING_TRANSIENT_STATE_KEYS)
));
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed");
return backgroundTaskRepository.save(task);
}
@Transactional
public int requeueExpiredRunningTasks() {
LocalDateTime now = LocalDateTime.now();
int recovered = 0;
for (Long taskId : backgroundTaskRepository.findExpiredRunningTaskIds(
BackgroundTaskStatus.RUNNING,
now,
PageRequest.of(0, EXPIRED_RUNNING_TASK_BATCH_SIZE)
)) {
int requeued = backgroundTaskRepository.requeueExpiredRunningTask(
taskId,
BackgroundTaskStatus.RUNNING,
BackgroundTaskStatus.QUEUED,
now,
now
);
if (requeued != 1) {
continue;
}
BackgroundTask task = backgroundTaskRepository.findById(taskId)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
resetTaskToQueued(task);
backgroundTaskRepository.save(task);
recovered += 1;
}
return recovered;
}
public List<Long> findQueuedTaskIds(int limit) {
if (limit <= 0) {
return List.of();
}
return backgroundTaskRepository.findReadyTaskIdsByStatusOrder(
BackgroundTaskStatus.QUEUED,
LocalDateTime.now(),
PageRequest.of(0, limit)
);
}
@Transactional
public Optional<BackgroundTask> claimQueuedTask(Long id, String workerOwner, long leaseDurationSeconds) {
LocalDateTime now = LocalDateTime.now();
LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds));
int claimed = backgroundTaskRepository.claimQueuedTask(
id,
BackgroundTaskStatus.QUEUED,
BackgroundTaskStatus.RUNNING,
workerOwner,
leaseExpiresAt,
now,
now
);
if (claimed != 1) {
return Optional.empty();
}
Optional<BackgroundTask> task = backgroundTaskRepository.findById(id);
task.ifPresent(claimedTask -> {
claimedTask.setLeaseOwner(workerOwner);
claimedTask.setLeaseExpiresAt(leaseExpiresAt);
claimedTask.setHeartbeatAt(now);
claimedTask.setPublicStateJson(mergePublicStateJson(
claimedTask.getPublicStateJson(),
runningStatePatch(claimedTask, workerOwner, now, leaseExpiresAt, true),
RETRY_TRANSIENT_STATE_KEYS
));
});
task.ifPresent(backgroundTaskRepository::save);
return task;
}
@Transactional
public BackgroundTask markWorkerTaskProgress(Long id,
String workerOwner,
Map<String, Object> publicStatePatch,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
task.setLeaseOwner(workerOwner);
task.setLeaseExpiresAt(leaseTouch.leaseExpiresAt());
task.setHeartbeatAt(leaseTouch.now());
Map<String, Object> nextPatch = new LinkedHashMap<>(runningStatePatch(
task,
workerOwner,
leaseTouch.now(),
leaseTouch.leaseExpiresAt(),
false
));
if (publicStatePatch != null) {
nextPatch.putAll(publicStatePatch);
}
task.setPublicStateJson(mergePublicStateJson(task.getPublicStateJson(), nextPatch));
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markWorkerTaskCompleted(Long id,
String workerOwner,
Map<String, Object> publicStatePatch,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
Map<String, Object> nextPatch = new LinkedHashMap<>(publicStatePatch == null ? Map.of() : publicStatePatch);
nextPatch.put(STATE_PHASE_KEY, "completed");
nextPatch.put(STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount());
nextPatch.put(STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts());
nextPatch.put(STATE_HEARTBEAT_AT_KEY, leaseTouch.now().toString());
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
nextPatch,
removableStateKeys(RETRY_TRANSIENT_STATE_KEYS, RUNNING_TRANSIENT_STATE_KEYS)
));
task.setStatus(BackgroundTaskStatus.COMPLETED);
task.setNextRunAt(null);
clearLease(task);
task.setFinishedAt(LocalDateTime.now());
task.setErrorMessage(null);
return backgroundTaskRepository.save(task);
}
@Transactional
public BackgroundTask markWorkerTaskFailed(Long id,
String workerOwner,
String errorMessage,
BackgroundTaskFailureCategory failureCategory,
long leaseDurationSeconds) {
LeaseTouch leaseTouch = refreshLease(id, workerOwner, leaseDurationSeconds);
BackgroundTask task = backgroundTaskRepository.findById(id)
.orElseThrow(() -> new ApiV2Exception(ApiV2ErrorCode.FILE_NOT_FOUND, "task not found"));
String normalizedErrorMessage = StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed"; String normalizedErrorMessage = StringUtils.hasText(errorMessage) ? errorMessage.trim() : "task failed";
LocalDateTime now = leaseTouch.now(); task.setPublicStateJson(stateManager.merge(
if (failureCategory.isRetryable() && hasRemainingAttempts(task)) {
long retryDelaySeconds = resolveRetryDelaySeconds(task.getType(), failureCategory, task.getAttemptCount());
LocalDateTime nextRunAt = now.plusSeconds(retryDelaySeconds);
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setNextRunAt(nextRunAt);
clearLease(task);
task.setFinishedAt(null);
task.setErrorMessage(null);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(), task.getPublicStateJson(),
Map.of( stateManager.failedStatePatch(
STATE_PHASE_KEY, "queued", task,
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(), normalizedErrorMessage,
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(), BackgroundTaskFailureCategory.UNKNOWN,
STATE_RETRY_SCHEDULED_KEY, true, LocalDateTime.now()
STATE_NEXT_RETRY_AT_KEY, nextRunAt.toString(),
STATE_RETRY_DELAY_SECONDS_KEY, retryDelaySeconds,
STATE_LAST_FAILURE_MESSAGE_KEY, normalizedErrorMessage,
STATE_LAST_FAILURE_AT_KEY, now.toString(),
STATE_FAILURE_CATEGORY_KEY, failureCategory.name(),
STATE_HEARTBEAT_AT_KEY, now.toString()
), ),
RUNNING_TRANSIENT_STATE_KEYS stateManager.removableKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY), RUNNING_TRANSIENT_STATE_KEYS)
)); ));
return backgroundTaskRepository.save(task); task.setFinishedAt(LocalDateTime.now());
}
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(mergePublicStateJson(
task.getPublicStateJson(),
Map.of(
STATE_PHASE_KEY, "failed",
STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount(),
STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts(),
STATE_LAST_FAILURE_MESSAGE_KEY, normalizedErrorMessage,
STATE_LAST_FAILURE_AT_KEY, now.toString(),
STATE_FAILURE_CATEGORY_KEY, failureCategory.name(),
STATE_HEARTBEAT_AT_KEY, now.toString()
),
removableStateKeys(List.of(STATE_RETRY_SCHEDULED_KEY, STATE_NEXT_RETRY_AT_KEY, STATE_RETRY_DELAY_SECONDS_KEY), RUNNING_TRANSIENT_STATE_KEYS)
));
task.setStatus(BackgroundTaskStatus.FAILED);
task.setFinishedAt(now);
task.setErrorMessage(normalizedErrorMessage); task.setErrorMessage(normalizedErrorMessage);
return backgroundTaskRepository.save(task); return backgroundTaskRepository.save(task);
} }
@@ -436,6 +297,10 @@ public class BackgroundTaskService {
return UUID.randomUUID().toString().replace("-", ""); return UUID.randomUUID().toString().replace("-", "");
} }
private String correlationLockName(String correlationId) {
return "background-task-correlation:" + correlationId;
}
private void validateTaskTarget(BackgroundTaskType type, StoredFile file) { private void validateTaskTarget(BackgroundTaskType type, StoredFile file) {
if (type == BackgroundTaskType.ARCHIVE) { if (type == BackgroundTaskType.ARCHIVE) {
return; return;
@@ -446,7 +311,8 @@ public class BackgroundTaskService {
if (type == BackgroundTaskType.EXTRACT && !isZipCompatibleArchive(file)) { if (type == BackgroundTaskType.EXTRACT && !isZipCompatibleArchive(file)) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "extract task only supports zip-compatible archives"); throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "extract task only supports zip-compatible archives");
} }
if (type == BackgroundTaskType.MEDIA_META && !isMediaLike(file)) { if (type == BackgroundTaskType.MEDIA_META
&& !MediaTaskSupport.isMediaLike(file.getFilename(), file.getContentType())) {
throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "media metadata task only supports media files"); throw new ApiV2Exception(ApiV2ErrorCode.BAD_REQUEST, "media metadata task only supports media files");
} }
} }
@@ -470,14 +336,6 @@ public class BackgroundTaskService {
return hasExtension(file.getFilename(), ZIP_COMPATIBLE_EXTENSIONS); return hasExtension(file.getFilename(), ZIP_COMPATIBLE_EXTENSIONS);
} }
private boolean isMediaLike(StoredFile file) {
String contentType = normalizeContentType(file.getContentType());
if (contentType.startsWith("image/") || contentType.startsWith("video/") || contentType.startsWith("audio/")) {
return true;
}
return hasExtension(file.getFilename(), MEDIA_EXTENSIONS);
}
private String deriveExtractOutputDirectoryName(String filename) { private String deriveExtractOutputDirectoryName(String filename) {
if (!StringUtils.hasText(filename)) { if (!StringUtils.hasText(filename)) {
return "extracted"; return "extracted";
@@ -511,6 +369,42 @@ public class BackgroundTaskService {
return contentType.trim().toLowerCase(Locale.ROOT); return contentType.trim().toLowerCase(Locale.ROOT);
} }
private BackgroundTask createQueuedFileTaskInternal(Long userId,
BackgroundTaskType type,
StoredFile file,
String correlationId) {
return createQueuedFileTaskInternal(userId, type, file, correlationId, false);
}
private BackgroundTask createQueuedFileTaskInternal(Long userId,
BackgroundTaskType type,
StoredFile file,
String correlationId,
boolean flushOnSave) {
String logicalPath = buildLogicalPath(file);
validateTaskTarget(type, file);
Map<String, Object> publicState = fileState(file, logicalPath);
Map<String, Object> privateState = new LinkedHashMap<>(publicState);
privateState.put("taskType", type.name());
if (type == BackgroundTaskType.ARCHIVE) {
String outputPath = file.getPath();
String outputFilename = file.getFilename() + ".zip";
publicState.put("outputPath", outputPath);
publicState.put("outputFilename", outputFilename);
privateState.put("outputPath", outputPath);
privateState.put("outputFilename", outputFilename);
} else if (type == BackgroundTaskType.EXTRACT) {
String outputPath = file.getPath();
String outputDirectoryName = deriveExtractOutputDirectoryName(file.getFilename());
publicState.put("outputPath", outputPath);
publicState.put("outputDirectoryName", outputDirectoryName);
privateState.put("outputPath", outputPath);
privateState.put("outputDirectoryName", outputDirectoryName);
}
return createQueuedTask(userId, type, publicState, privateState, correlationId, flushOnSave);
}
private String buildLogicalPath(StoredFile file) { private String buildLogicalPath(StoredFile file) {
String parent = normalizeLogicalPath(file.getPath()); String parent = normalizeLogicalPath(file.getPath());
if ("/".equals(parent)) { if ("/".equals(parent)) {
@@ -536,148 +430,9 @@ public class BackgroundTaskService {
return normalized; return normalized;
} }
private String toJson(Map<String, Object> value) {
Map<String, Object> safeValue = value == null ? new LinkedHashMap<>() : new LinkedHashMap<>(value);
try {
return objectMapper.writeValueAsString(safeValue);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to serialize background task state", ex);
}
}
private Map<String, Object> parseJsonObject(String value) {
if (!StringUtils.hasText(value)) {
return new LinkedHashMap<>();
}
try {
return objectMapper.readValue(value, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to parse background task state", ex);
}
}
private String mergePublicStateJson(String currentValue, Map<String, Object> patch) {
return mergePublicStateJson(currentValue, patch, List.of());
}
private String mergePublicStateJson(String currentValue, Map<String, Object> patch, List<String> keysToRemove) {
Map<String, Object> nextPublicState = parseJsonObject(currentValue);
if (keysToRemove != null) {
keysToRemove.forEach(nextPublicState::remove);
}
if (patch != null) {
nextPublicState.putAll(patch);
}
return toJson(nextPublicState);
}
private String resetPublicStateForRetry(String privateStateJson, int attemptCount, int maxAttempts) {
Map<String, Object> nextPublicState = parseJsonObject(privateStateJson);
nextPublicState.remove("taskType");
nextPublicState.put(STATE_PHASE_KEY, "queued");
nextPublicState.putAll(retryStatePatch(attemptCount, maxAttempts));
return toJson(nextPublicState);
}
private void resetTaskToQueued(BackgroundTask task) {
task.setNextRunAt(null);
clearLease(task);
task.setPublicStateJson(resetPublicStateForRetry(task.getPrivateStateJson(), task.getAttemptCount(), task.getMaxAttempts()));
task.setStatus(BackgroundTaskStatus.QUEUED);
task.setFinishedAt(null);
task.setErrorMessage(null);
}
private int resolveMaxAttempts(BackgroundTaskType type) {
return switch (type) {
case ARCHIVE -> 4;
case EXTRACT -> 3;
case MEDIA_META -> 2;
default -> 1;
};
}
private Map<String, Object> retryStatePatch(int attemptCount, int maxAttempts) {
Map<String, Object> patch = new LinkedHashMap<>();
patch.put(STATE_ATTEMPT_COUNT_KEY, attemptCount);
patch.put(STATE_MAX_ATTEMPTS_KEY, maxAttempts);
return patch;
}
private boolean hasRemainingAttempts(BackgroundTask task) {
return task.getAttemptCount() != null
&& task.getMaxAttempts() != null
&& task.getAttemptCount() < task.getMaxAttempts();
}
private long resolveRetryDelaySeconds(BackgroundTaskType type,
BackgroundTaskFailureCategory failureCategory,
Integer attemptCount) {
int safeAttemptCount = attemptCount == null ? 1 : Math.max(1, attemptCount);
long baseDelaySeconds = switch (type) {
case ARCHIVE -> 30L;
case EXTRACT -> 45L;
case MEDIA_META -> 15L;
default -> 30L;
};
if (failureCategory == BackgroundTaskFailureCategory.RATE_LIMITED) {
baseDelaySeconds *= 4L;
} else if (failureCategory == BackgroundTaskFailureCategory.UNKNOWN) {
baseDelaySeconds *= 2L;
}
long delay = baseDelaySeconds * (1L << Math.min(safeAttemptCount - 1, 2));
return Math.min(delay, baseDelaySeconds * 4L);
}
private LeaseTouch refreshLease(Long id, String workerOwner, long leaseDurationSeconds) {
LocalDateTime now = LocalDateTime.now();
LocalDateTime leaseExpiresAt = now.plusSeconds(Math.max(30L, leaseDurationSeconds));
int refreshed = backgroundTaskRepository.refreshRunningTaskLease(
id,
BackgroundTaskStatus.RUNNING,
workerOwner,
leaseExpiresAt,
now,
now
);
if (refreshed != 1) {
throw new BackgroundTaskLeaseLostException(id, workerOwner);
}
return new LeaseTouch(now, leaseExpiresAt);
}
private Map<String, Object> runningStatePatch(BackgroundTask task,
String workerOwner,
LocalDateTime heartbeatAt,
LocalDateTime leaseExpiresAt,
boolean includeStartedAt) {
Map<String, Object> patch = new LinkedHashMap<>();
patch.put(STATE_PHASE_KEY, "running");
patch.put(STATE_ATTEMPT_COUNT_KEY, task.getAttemptCount());
patch.put(STATE_MAX_ATTEMPTS_KEY, task.getMaxAttempts());
patch.put(STATE_WORKER_OWNER_KEY, workerOwner);
patch.put(STATE_HEARTBEAT_AT_KEY, heartbeatAt.toString());
patch.put(STATE_LEASE_EXPIRES_AT_KEY, leaseExpiresAt.toString());
if (includeStartedAt) {
patch.put(STATE_STARTED_AT_KEY, heartbeatAt.toString());
}
return patch;
}
private List<String> removableStateKeys(List<String> primary, List<String> secondary) {
List<String> keys = new java.util.ArrayList<>(primary);
keys.addAll(secondary);
return keys;
}
private void clearLease(BackgroundTask task) { private void clearLease(BackgroundTask task) {
task.setLeaseOwner(null); task.setLeaseOwner(null);
task.setLeaseExpiresAt(null); task.setLeaseExpiresAt(null);
task.setHeartbeatAt(null); task.setHeartbeatAt(null);
} }
private record LeaseTouch(LocalDateTime now, LocalDateTime leaseExpiresAt) {
}
} }

View File

@@ -11,11 +11,11 @@ import org.springframework.stereotype.Component;
@Slf4j @Slf4j
public class BackgroundTaskStartupRecovery { public class BackgroundTaskStartupRecovery {
private final BackgroundTaskService backgroundTaskService; private final BackgroundTaskExecutionService backgroundTaskExecutionService;
@EventListener(ApplicationReadyEvent.class) @EventListener(ApplicationReadyEvent.class)
public void recoverOnStartup() { public void recoverOnStartup() {
int recovered = backgroundTaskService.requeueExpiredRunningTasks(); int recovered = backgroundTaskExecutionService.requeueExpiredRunningTasks();
if (recovered > 0) { if (recovered > 0) {
log.warn("Recovered {} expired RUNNING background task leases back to QUEUED on startup", recovered); log.warn("Recovered {} expired RUNNING background task leases back to QUEUED on startup", recovered);
} }

View File

@@ -0,0 +1,21 @@
package com.yoyuzh.files.tasks;
public final class BackgroundTaskStateKeys {
public static final String PHASE = "phase";
public static final String ATTEMPT_COUNT = "attemptCount";
public static final String MAX_ATTEMPTS = "maxAttempts";
public static final String RETRY_SCHEDULED = "retryScheduled";
public static final String NEXT_RETRY_AT = "nextRetryAt";
public static final String RETRY_DELAY_SECONDS = "retryDelaySeconds";
public static final String LAST_FAILURE_MESSAGE = "lastFailureMessage";
public static final String LAST_FAILURE_AT = "lastFailureAt";
public static final String FAILURE_CATEGORY = "failureCategory";
public static final String WORKER_OWNER = "workerOwner";
public static final String HEARTBEAT_AT = "heartbeatAt";
public static final String LEASE_EXPIRES_AT = "leaseExpiresAt";
public static final String STARTED_AT = "startedAt";
private BackgroundTaskStateKeys() {
}
}

View File

@@ -0,0 +1,185 @@
package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Component
public class BackgroundTaskStateManager {
private static final TypeReference<LinkedHashMap<String, Object>> JSON_OBJECT_TYPE = new TypeReference<>() {
};
private final ObjectMapper objectMapper;
public BackgroundTaskStateManager(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
public String toJson(Map<String, Object> value) {
Map<String, Object> safeValue = value == null ? new LinkedHashMap<>() : new LinkedHashMap<>(value);
try {
return objectMapper.writeValueAsString(safeValue);
} catch (JsonProcessingException ex) {
throw new IllegalStateException("Failed to serialize background task state", ex);
}
}
public Map<String, Object> retryStatePatch(int attemptCount, int maxAttempts) {
Map<String, Object> patch = new LinkedHashMap<>();
patch.put(BackgroundTaskStateKeys.ATTEMPT_COUNT, attemptCount);
patch.put(BackgroundTaskStateKeys.MAX_ATTEMPTS, maxAttempts);
return patch;
}
public Map<String, Object> runningStatePatch(BackgroundTask task,
String workerOwner,
LocalDateTime heartbeatAt,
LocalDateTime leaseExpiresAt,
boolean includeStartedAt) {
Map<String, Object> patch = new LinkedHashMap<>();
patch.put(BackgroundTaskStateKeys.PHASE, "running");
patch.put(BackgroundTaskStateKeys.ATTEMPT_COUNT, task.getAttemptCount());
patch.put(BackgroundTaskStateKeys.MAX_ATTEMPTS, task.getMaxAttempts());
patch.put(BackgroundTaskStateKeys.WORKER_OWNER, workerOwner);
patch.put(BackgroundTaskStateKeys.HEARTBEAT_AT, heartbeatAt.toString());
patch.put(BackgroundTaskStateKeys.LEASE_EXPIRES_AT, leaseExpiresAt.toString());
if (includeStartedAt) {
patch.put(BackgroundTaskStateKeys.STARTED_AT, heartbeatAt.toString());
}
return patch;
}
public Map<String, Object> cancelledStatePatch(BackgroundTask task, LocalDateTime heartbeatAt) {
return terminalStatePatch("cancelled", task, heartbeatAt);
}
public Map<String, Object> completedStatePatch(BackgroundTask task,
LocalDateTime heartbeatAt,
Map<String, Object> additionalPatch) {
Map<String, Object> patch = new LinkedHashMap<>(additionalPatch == null ? Map.of() : additionalPatch);
patch.putAll(terminalStatePatch("completed", task, heartbeatAt));
return patch;
}
public Map<String, Object> failedStatePatch(BackgroundTask task,
String errorMessage,
BackgroundTaskFailureCategory failureCategory,
LocalDateTime heartbeatAt) {
Map<String, Object> patch = new LinkedHashMap<>(terminalStatePatch("failed", task, heartbeatAt));
patch.put(BackgroundTaskStateKeys.LAST_FAILURE_MESSAGE, errorMessage);
patch.put(BackgroundTaskStateKeys.LAST_FAILURE_AT, heartbeatAt.toString());
patch.put(BackgroundTaskStateKeys.FAILURE_CATEGORY, failureCategory.name());
return patch;
}
public Map<String, Object> retryQueuedStatePatch(BackgroundTask task,
String errorMessage,
BackgroundTaskFailureCategory failureCategory,
LocalDateTime nextRetryAt,
long retryDelaySeconds,
LocalDateTime heartbeatAt) {
Map<String, Object> patch = new LinkedHashMap<>(retryStatePatch(task.getAttemptCount(), task.getMaxAttempts()));
patch.put(BackgroundTaskStateKeys.PHASE, "queued");
patch.put(BackgroundTaskStateKeys.RETRY_SCHEDULED, true);
patch.put(BackgroundTaskStateKeys.NEXT_RETRY_AT, nextRetryAt.toString());
patch.put(BackgroundTaskStateKeys.RETRY_DELAY_SECONDS, retryDelaySeconds);
patch.put(BackgroundTaskStateKeys.LAST_FAILURE_MESSAGE, errorMessage);
patch.put(BackgroundTaskStateKeys.LAST_FAILURE_AT, heartbeatAt.toString());
patch.put(BackgroundTaskStateKeys.FAILURE_CATEGORY, failureCategory.name());
patch.put(BackgroundTaskStateKeys.HEARTBEAT_AT, heartbeatAt.toString());
return patch;
}
public String createInitialPublicState(Map<String, Object> baseState, int attemptCount, int maxAttempts) {
Map<String, Object> nextPublicState = new LinkedHashMap<>(baseState == null ? Map.of() : baseState);
nextPublicState.put(BackgroundTaskStateKeys.PHASE, "queued");
nextPublicState.putAll(retryStatePatch(attemptCount, maxAttempts));
return toJson(nextPublicState);
}
public String merge(String currentValue, Map<String, Object> patch) {
return merge(currentValue, patch, List.of());
}
public String merge(String currentValue, Map<String, Object> patch, List<String> keysToRemove) {
Map<String, Object> nextPublicState = parse(currentValue);
if (keysToRemove != null) {
keysToRemove.forEach(nextPublicState::remove);
}
if (patch != null) {
nextPublicState.putAll(patch);
}
return toJson(nextPublicState);
}
public String resetPublicStateForRetry(String privateStateJson, int attemptCount, int maxAttempts) {
Map<String, Object> nextPublicState = parse(privateStateJson);
nextPublicState.remove("taskType");
nextPublicState.put(BackgroundTaskStateKeys.PHASE, "queued");
nextPublicState.putAll(retryStatePatch(attemptCount, maxAttempts));
return toJson(nextPublicState);
}
public List<String> removableKeys(List<String> primary, List<String> secondary) {
List<String> keys = new java.util.ArrayList<>(primary);
keys.addAll(secondary);
return keys;
}
public Map<String, Object> parseJsonObject(String value, String invalidStateMessage) {
if (!StringUtils.hasText(value)) {
return Map.of();
}
try {
return objectMapper.readValue(value, JSON_OBJECT_TYPE);
} catch (JsonProcessingException ex) {
throw new IllegalStateException(invalidStateMessage, ex);
}
}
public Map<String, Object> mergeJsonObjects(String primaryJson,
String overlayJson,
String invalidStateMessage) {
Map<String, Object> state = new LinkedHashMap<>(parseJsonObject(primaryJson, invalidStateMessage));
state.putAll(parseJsonObject(overlayJson, invalidStateMessage));
return state;
}
public Long readLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text && StringUtils.hasText(text)) {
return Long.parseLong(text.trim());
}
return null;
}
public String readText(Object value) {
if (value instanceof String text && StringUtils.hasText(text)) {
return text.trim();
}
return null;
}
private Map<String, Object> terminalStatePatch(String phase,
BackgroundTask task,
LocalDateTime heartbeatAt) {
Map<String, Object> patch = new LinkedHashMap<>(retryStatePatch(task.getAttemptCount(), task.getMaxAttempts()));
patch.put(BackgroundTaskStateKeys.PHASE, phase);
patch.put(BackgroundTaskStateKeys.HEARTBEAT_AT, heartbeatAt.toString());
return patch;
}
private Map<String, Object> parse(String value) {
return new LinkedHashMap<>(parseJsonObject(value, "Failed to parse background task state"));
}
}

View File

@@ -19,13 +19,13 @@ public class BackgroundTaskWorker {
private static final int DEFAULT_BATCH_SIZE = 5; private static final int DEFAULT_BATCH_SIZE = 5;
private static final long DEFAULT_LEASE_DURATION_SECONDS = 120L; private static final long DEFAULT_LEASE_DURATION_SECONDS = 120L;
private final BackgroundTaskService backgroundTaskService; private final BackgroundTaskExecutionService backgroundTaskExecutionService;
private final List<BackgroundTaskHandler> handlers; private final List<BackgroundTaskHandler> handlers;
private final String workerOwner; private final String workerOwner;
public BackgroundTaskWorker(BackgroundTaskService backgroundTaskService, public BackgroundTaskWorker(BackgroundTaskExecutionService backgroundTaskExecutionService,
List<BackgroundTaskHandler> handlers) { List<BackgroundTaskHandler> handlers) {
this.backgroundTaskService = backgroundTaskService; this.backgroundTaskExecutionService = backgroundTaskExecutionService;
this.handlers = List.copyOf(handlers); this.handlers = List.copyOf(handlers);
this.workerOwner = UUID.randomUUID().toString().replace("-", ""); this.workerOwner = UUID.randomUUID().toString().replace("-", "");
} }
@@ -39,10 +39,10 @@ public class BackgroundTaskWorker {
} }
public int processQueuedTasks(int maxTasks) { public int processQueuedTasks(int maxTasks) {
backgroundTaskService.requeueExpiredRunningTasks(); backgroundTaskExecutionService.requeueExpiredRunningTasks();
int processedCount = 0; int processedCount = 0;
for (Long taskId : backgroundTaskService.findQueuedTaskIds(maxTasks)) { for (Long taskId : backgroundTaskExecutionService.findQueuedTaskIds(maxTasks)) {
var claimedTask = backgroundTaskService.claimQueuedTask(taskId, workerOwner, DEFAULT_LEASE_DURATION_SECONDS); var claimedTask = backgroundTaskExecutionService.claimQueuedTask(taskId, workerOwner, DEFAULT_LEASE_DURATION_SECONDS);
if (claimedTask.isEmpty()) { if (claimedTask.isEmpty()) {
continue; continue;
} }
@@ -55,21 +55,21 @@ public class BackgroundTaskWorker {
private void execute(BackgroundTask task) { private void execute(BackgroundTask task) {
try { try {
backgroundTaskService.markWorkerTaskProgress( backgroundTaskExecutionService.markWorkerTaskProgress(
task.getId(), task.getId(),
workerOwner, workerOwner,
Map.of(BackgroundTaskService.STATE_PHASE_KEY, resolveRunningPhase(task.getType())), Map.of(BackgroundTaskStateKeys.PHASE, resolveRunningPhase(task.getType())),
DEFAULT_LEASE_DURATION_SECONDS DEFAULT_LEASE_DURATION_SECONDS
); );
BackgroundTaskHandler handler = findHandler(task); BackgroundTaskHandler handler = findHandler(task);
BackgroundTaskHandlerResult result = handler.handle(task, publicStatePatch -> BackgroundTaskHandlerResult result = handler.handle(task, publicStatePatch ->
backgroundTaskService.markWorkerTaskProgress( backgroundTaskExecutionService.markWorkerTaskProgress(
task.getId(), task.getId(),
workerOwner, workerOwner,
publicStatePatch, publicStatePatch,
DEFAULT_LEASE_DURATION_SECONDS DEFAULT_LEASE_DURATION_SECONDS
)); ));
backgroundTaskService.markWorkerTaskCompleted( backgroundTaskExecutionService.markWorkerTaskCompleted(
task.getId(), task.getId(),
workerOwner, workerOwner,
result.publicStatePatch(), result.publicStatePatch(),
@@ -80,7 +80,7 @@ public class BackgroundTaskWorker {
} catch (Exception ex) { } catch (Exception ex) {
String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage(); String message = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage();
try { try {
backgroundTaskService.markWorkerTaskFailed( backgroundTaskExecutionService.markWorkerTaskFailed(
task.getId(), task.getId(),
workerOwner, workerOwner,
message, message,

View File

@@ -1,8 +1,5 @@
package com.yoyuzh.files.tasks; package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.auth.User; import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository; import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
@@ -28,16 +25,16 @@ public class ExtractBackgroundTaskHandler implements BackgroundTaskHandler {
private final StoredFileRepository storedFileRepository; private final StoredFileRepository storedFileRepository;
private final UserRepository userRepository; private final UserRepository userRepository;
private final FileService fileService; private final FileService fileService;
private final ObjectMapper objectMapper; private final BackgroundTaskStateManager stateManager;
public ExtractBackgroundTaskHandler(StoredFileRepository storedFileRepository, public ExtractBackgroundTaskHandler(StoredFileRepository storedFileRepository,
UserRepository userRepository, UserRepository userRepository,
FileService fileService, FileService fileService,
ObjectMapper objectMapper) { BackgroundTaskStateManager stateManager) {
this.storedFileRepository = storedFileRepository; this.storedFileRepository = storedFileRepository;
this.userRepository = userRepository; this.userRepository = userRepository;
this.fileService = fileService; this.fileService = fileService;
this.objectMapper = objectMapper; this.stateManager = stateManager;
} }
@Override @Override
@@ -53,10 +50,14 @@ public class ExtractBackgroundTaskHandler implements BackgroundTaskHandler {
@Override @Override
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) { public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
Map<String, Object> state = parseState(task.getPrivateStateJson(), task.getPublicStateJson()); Map<String, Object> state = stateManager.mergeJsonObjects(
Long fileId = extractLong(state.get("fileId")); task.getPublicStateJson(),
String outputPath = extractText(state.get("outputPath")); task.getPrivateStateJson(),
String outputDirectoryName = extractText(state.get("outputDirectoryName")); "extract task state is invalid"
);
Long fileId = stateManager.readLong(state.get("fileId"));
String outputPath = stateManager.readText(state.get("outputPath"));
String outputDirectoryName = stateManager.readText(state.get("outputDirectoryName"));
if (fileId == null) { if (fileId == null) {
throw new IllegalStateException("extract task missing fileId"); throw new IllegalStateException("extract task missing fileId");
} }
@@ -235,41 +236,6 @@ public class ExtractBackgroundTaskHandler implements BackgroundTaskHandler {
return relativePath; return relativePath;
} }
private Map<String, Object> parseState(String privateStateJson, String publicStateJson) {
Map<String, Object> state = new LinkedHashMap<>(parseJsonObject(publicStateJson));
state.putAll(parseJsonObject(privateStateJson));
return state;
}
private Map<String, Object> parseJsonObject(String json) {
if (!StringUtils.hasText(json)) {
return Map.of();
}
try {
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("extract task state is invalid", ex);
}
}
private Long extractLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text && StringUtils.hasText(text)) {
return Long.parseLong(text.trim());
}
return null;
}
private String extractText(Object value) {
if (value instanceof String text && StringUtils.hasText(text)) {
return text.trim();
}
return null;
}
private String normalizeDirectoryPath(String path) { private String normalizeDirectoryPath(String path) {
if (!StringUtils.hasText(path)) { if (!StringUtils.hasText(path)) {
return "/"; return "/";

View File

@@ -1,8 +1,5 @@
package com.yoyuzh.files.tasks; package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.files.core.FileBlob; import com.yoyuzh.files.core.FileBlob;
import com.yoyuzh.files.core.StoredFile; import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
@@ -18,7 +15,6 @@ import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
@@ -34,16 +30,16 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
private final StoredFileRepository storedFileRepository; private final StoredFileRepository storedFileRepository;
private final FileMetadataRepository fileMetadataRepository; private final FileMetadataRepository fileMetadataRepository;
private final FileContentStorage fileContentStorage; private final FileContentStorage fileContentStorage;
private final ObjectMapper objectMapper; private final BackgroundTaskStateManager stateManager;
public MediaMetadataBackgroundTaskHandler(StoredFileRepository storedFileRepository, public MediaMetadataBackgroundTaskHandler(StoredFileRepository storedFileRepository,
FileMetadataRepository fileMetadataRepository, FileMetadataRepository fileMetadataRepository,
FileContentStorage fileContentStorage, FileContentStorage fileContentStorage,
ObjectMapper objectMapper) { BackgroundTaskStateManager stateManager) {
this.storedFileRepository = storedFileRepository; this.storedFileRepository = storedFileRepository;
this.fileMetadataRepository = fileMetadataRepository; this.fileMetadataRepository = fileMetadataRepository;
this.fileContentStorage = fileContentStorage; this.fileContentStorage = fileContentStorage;
this.objectMapper = objectMapper; this.stateManager = stateManager;
} }
@Override @Override
@@ -114,39 +110,21 @@ public class MediaMetadataBackgroundTaskHandler implements BackgroundTaskHandler
} }
private Long readFileId(BackgroundTask task) { private Long readFileId(BackgroundTask task) {
Long fileId = extractLong(parseState(task.getPrivateStateJson()).get("fileId")); Long fileId = stateManager.readLong(
stateManager.parseJsonObject(task.getPrivateStateJson(), "media metadata task state is invalid").get("fileId")
);
if (fileId != null) { if (fileId != null) {
return fileId; return fileId;
} }
fileId = extractLong(parseState(task.getPublicStateJson()).get("fileId")); fileId = stateManager.readLong(
stateManager.parseJsonObject(task.getPublicStateJson(), "media metadata task state is invalid").get("fileId")
);
if (fileId != null) { if (fileId != null) {
return fileId; return fileId;
} }
throw new IllegalStateException("media metadata task missing fileId"); throw new IllegalStateException("media metadata task missing fileId");
} }
private Map<String, Object> parseState(String json) {
if (!StringUtils.hasText(json)) {
return Map.of();
}
try {
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("media metadata task state is invalid", ex);
}
}
private Long extractLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text && StringUtils.hasText(text)) {
return Long.parseLong(text.trim());
}
return null;
}
private String firstText(String primary, String fallback) { private String firstText(String primary, String fallback) {
if (StringUtils.hasText(primary)) { if (StringUtils.hasText(primary)) {
return primary.trim(); return primary.trim();

View File

@@ -0,0 +1,83 @@
package com.yoyuzh.files.tasks;
import com.yoyuzh.common.broker.LightweightBrokerService;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.util.Map;
@Component
public class MediaMetadataTaskBrokerConsumer {
private static final int DEFAULT_BATCH_SIZE = 10;
private final LightweightBrokerService lightweightBrokerService;
private final BackgroundTaskCommandService backgroundTaskCommandService;
public MediaMetadataTaskBrokerConsumer(LightweightBrokerService lightweightBrokerService,
BackgroundTaskCommandService backgroundTaskCommandService) {
this.lightweightBrokerService = lightweightBrokerService;
this.backgroundTaskCommandService = backgroundTaskCommandService;
}
@Scheduled(
fixedDelayString = "${app.redis.broker.media-meta.fixed-delay-ms:3000}",
initialDelayString = "${app.redis.broker.media-meta.initial-delay-ms:15000}"
)
public void runScheduledBatch() {
drainQueuedMessages(DEFAULT_BATCH_SIZE);
}
public int drainQueuedMessages(int maxMessages) {
int safeLimit = Math.max(0, maxMessages);
int processed = 0;
for (int i = 0; i < safeLimit; i++) {
var payload = lightweightBrokerService.poll(MediaMetadataTaskBrokerPublisher.TOPIC);
if (payload.isEmpty()) {
break;
}
try {
if (handlePayload(payload.get())) {
processed += 1;
}
} catch (RuntimeException ex) {
lightweightBrokerService.requeue(MediaMetadataTaskBrokerPublisher.TOPIC, payload.get());
break;
}
}
return processed;
}
private boolean handlePayload(Map<String, Object> payload) {
Long userId = readLong(payload.get("userId"));
Long fileId = readLong(payload.get("fileId"));
String correlationId = readString(payload.get("correlationId"));
if (userId == null || fileId == null) {
return false;
}
backgroundTaskCommandService.createQueuedAutoMediaMetadataTask(userId, fileId, correlationId);
return true;
}
private Long readLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text && StringUtils.hasText(text)) {
try {
return Long.parseLong(text.trim());
} catch (NumberFormatException ex) {
return null;
}
}
return null;
}
private String readString(Object value) {
if (!(value instanceof String text) || !StringUtils.hasText(text)) {
return null;
}
return text.trim();
}
}

View File

@@ -0,0 +1,58 @@
package com.yoyuzh.files.tasks;
import com.yoyuzh.common.broker.LightweightBrokerService;
import com.yoyuzh.files.core.StoredFile;
import org.springframework.stereotype.Component;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.util.Map;
@Component
public class MediaMetadataTaskBrokerPublisher {
public static final String TOPIC = "media-metadata-trigger";
private final LightweightBrokerService lightweightBrokerService;
public MediaMetadataTaskBrokerPublisher(LightweightBrokerService lightweightBrokerService) {
this.lightweightBrokerService = lightweightBrokerService;
}
public void publishAfterCommit(StoredFile storedFile) {
if (!shouldPublish(storedFile)) {
return;
}
Runnable publishTask = () -> lightweightBrokerService.publish(TOPIC, Map.of(
"userId", storedFile.getUser().getId(),
"fileId", storedFile.getId(),
"correlationId", buildCorrelationId(storedFile)
));
if (TransactionSynchronizationManager.isActualTransactionActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
publishTask.run();
}
});
return;
}
publishTask.run();
}
private boolean shouldPublish(StoredFile storedFile) {
return storedFile != null
&& storedFile.getId() != null
&& storedFile.getUser() != null
&& storedFile.getUser().getId() != null
&& !storedFile.isDirectory()
&& MediaTaskSupport.isMediaLike(storedFile.getFilename(), storedFile.getContentType());
}
private String buildCorrelationId(StoredFile storedFile) {
return "media-meta:auto:file:" + storedFile.getId();
}
}

View File

@@ -0,0 +1,43 @@
package com.yoyuzh.files.tasks;
import org.springframework.util.StringUtils;
import java.util.List;
import java.util.Locale;
final class MediaTaskSupport {
private static final List<String> MEDIA_EXTENSIONS = List.of(
".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg",
".mp4", ".mov", ".mkv", ".webm", ".avi",
".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"
);
private MediaTaskSupport() {
}
static boolean isMediaLike(String filename, String contentType) {
String normalizedContentType = normalizeContentType(contentType);
if (normalizedContentType.startsWith("image/")
|| normalizedContentType.startsWith("video/")
|| normalizedContentType.startsWith("audio/")) {
return true;
}
return hasExtension(filename);
}
private static boolean hasExtension(String filename) {
if (!StringUtils.hasText(filename)) {
return false;
}
String normalized = filename.toLowerCase(Locale.ROOT);
return MEDIA_EXTENSIONS.stream().anyMatch(normalized::endsWith);
}
private static String normalizeContentType(String contentType) {
if (!StringUtils.hasText(contentType)) {
return "";
}
return contentType.trim().toLowerCase(Locale.ROOT);
}
}

View File

@@ -1,8 +1,5 @@
package com.yoyuzh.files.tasks; package com.yoyuzh.files.tasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.core.FileBlob; import com.yoyuzh.files.core.FileBlob;
@@ -40,20 +37,20 @@ public class StoragePolicyMigrationBackgroundTaskHandler implements BackgroundTa
private final FileBlobRepository fileBlobRepository; private final FileBlobRepository fileBlobRepository;
private final StoredFileRepository storedFileRepository; private final StoredFileRepository storedFileRepository;
private final FileContentStorage fileContentStorage; private final FileContentStorage fileContentStorage;
private final ObjectMapper objectMapper; private final BackgroundTaskStateManager stateManager;
public StoragePolicyMigrationBackgroundTaskHandler(StoragePolicyRepository storagePolicyRepository, public StoragePolicyMigrationBackgroundTaskHandler(StoragePolicyRepository storagePolicyRepository,
FileEntityRepository fileEntityRepository, FileEntityRepository fileEntityRepository,
FileBlobRepository fileBlobRepository, FileBlobRepository fileBlobRepository,
StoredFileRepository storedFileRepository, StoredFileRepository storedFileRepository,
FileContentStorage fileContentStorage, FileContentStorage fileContentStorage,
ObjectMapper objectMapper) { BackgroundTaskStateManager stateManager) {
this.storagePolicyRepository = storagePolicyRepository; this.storagePolicyRepository = storagePolicyRepository;
this.fileEntityRepository = fileEntityRepository; this.fileEntityRepository = fileEntityRepository;
this.fileBlobRepository = fileBlobRepository; this.fileBlobRepository = fileBlobRepository;
this.storedFileRepository = storedFileRepository; this.storedFileRepository = storedFileRepository;
this.fileContentStorage = fileContentStorage; this.fileContentStorage = fileContentStorage;
this.objectMapper = objectMapper; this.stateManager = stateManager;
} }
@Override @Override
@@ -69,7 +66,10 @@ public class StoragePolicyMigrationBackgroundTaskHandler implements BackgroundTa
@Override @Override
public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) { public BackgroundTaskHandlerResult handle(BackgroundTask task, BackgroundTaskProgressReporter progressReporter) {
Map<String, Object> state = parseState(task.getPrivateStateJson()); Map<String, Object> state = stateManager.parseJsonObject(
task.getPrivateStateJson(),
"storage policy migration task state is invalid"
);
Long sourcePolicyId = readLong(state.get("sourcePolicyId"), "sourcePolicyId"); Long sourcePolicyId = readLong(state.get("sourcePolicyId"), "sourcePolicyId");
Long targetPolicyId = readLong(state.get("targetPolicyId"), "targetPolicyId"); Long targetPolicyId = readLong(state.get("targetPolicyId"), "targetPolicyId");
@@ -210,7 +210,7 @@ public class StoragePolicyMigrationBackgroundTaskHandler implements BackgroundTa
String migrationStage, String migrationStage,
boolean migrationPerformed) { boolean migrationPerformed) {
Map<String, Object> patch = new LinkedHashMap<>(); Map<String, Object> patch = new LinkedHashMap<>();
patch.put(BackgroundTaskService.STATE_PHASE_KEY, "migrating-storage-policy"); patch.put(BackgroundTaskStateKeys.PHASE, "migrating-storage-policy");
patch.put("worker", "storage-policy-migration"); patch.put("worker", "storage-policy-migration");
patch.put("migrationStage", migrationStage); patch.put("migrationStage", migrationStage);
patch.put("migrationMode", migrationPerformed ? "executed" : "executing"); patch.put("migrationMode", migrationPerformed ? "executed" : "executing");
@@ -281,24 +281,10 @@ public class StoragePolicyMigrationBackgroundTaskHandler implements BackgroundTa
} }
} }
private Map<String, Object> parseState(String json) {
if (!StringUtils.hasText(json)) {
return Map.of();
}
try {
return objectMapper.readValue(json, new TypeReference<LinkedHashMap<String, Object>>() {
});
} catch (JsonProcessingException ex) {
throw new IllegalStateException("storage policy migration task state is invalid", ex);
}
}
private Long readLong(Object value, String key) { private Long readLong(Object value, String key) {
if (value instanceof Number number) { Long parsed = stateManager.readLong(value);
return number.longValue(); if (parsed != null) {
} return parsed;
if (value instanceof String text && StringUtils.hasText(text)) {
return Long.parseLong(text.trim());
} }
throw new IllegalStateException("storage policy migration task missing " + key); throw new IllegalStateException("storage policy migration task missing " + key);
} }

View File

@@ -0,0 +1,41 @@
package com.yoyuzh.files.upload;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.Optional;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "false", matchIfMissing = true)
public class NoOpUploadSessionRuntimeStateService implements UploadSessionRuntimeStateService {
@Override
public Optional<UploadSessionRuntimeState> getState(String sessionId) {
return Optional.empty();
}
@Override
public void markCreated(UploadSession session) {
}
@Override
public void markUploading(UploadSession session, long uploadedBytes, int uploadedPartCount, LocalDateTime updatedAt) {
}
@Override
public void markCompleted(UploadSession session, LocalDateTime updatedAt) {
}
@Override
public void markCancelled(UploadSession session, LocalDateTime updatedAt) {
}
@Override
public void markFailed(UploadSession session, LocalDateTime updatedAt) {
}
@Override
public void markExpired(UploadSession session, LocalDateTime updatedAt) {
}
}

View File

@@ -0,0 +1,160 @@
package com.yoyuzh.files.upload;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yoyuzh.config.AppRedisProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Optional;
@Service
@ConditionalOnProperty(prefix = "app.redis", name = "enabled", havingValue = "true")
public class RedisUploadSessionRuntimeStateService implements UploadSessionRuntimeStateService {
private final StringRedisTemplate stringRedisTemplate;
private final AppRedisProperties redisProperties;
private final ObjectMapper objectMapper;
public RedisUploadSessionRuntimeStateService(StringRedisTemplate stringRedisTemplate,
AppRedisProperties redisProperties,
ObjectMapper objectMapper) {
this.stringRedisTemplate = stringRedisTemplate;
this.redisProperties = redisProperties;
this.objectMapper = objectMapper;
}
@Override
public Optional<UploadSessionRuntimeState> getState(String sessionId) {
String value = stringRedisTemplate.opsForValue().get(buildKey(sessionId));
if (!StringUtils.hasText(value)) {
return Optional.empty();
}
try {
return Optional.of(objectMapper.readValue(value, UploadSessionRuntimeState.class));
} catch (JsonProcessingException ex) {
return Optional.empty();
}
}
@Override
public void markCreated(UploadSession session) {
LocalDateTime updatedAt = safeUpdatedAt(session);
writeState(session, new UploadSessionRuntimeState(
"created",
0L,
0,
0,
updatedAt,
session.getExpiresAt()
));
}
@Override
public void markUploading(UploadSession session, long uploadedBytes, int uploadedPartCount, LocalDateTime updatedAt) {
writeState(session, new UploadSessionRuntimeState(
"uploading",
Math.max(0L, uploadedBytes),
Math.max(0, uploadedPartCount),
toProgressPercent(uploadedBytes, session.getSize()),
updatedAt,
session.getExpiresAt()
));
}
@Override
public void markCompleted(UploadSession session, LocalDateTime updatedAt) {
writeState(session, new UploadSessionRuntimeState(
"completed",
session.getSize() == null ? 0L : session.getSize(),
Math.max(1, session.getChunkCount() == null ? 1 : session.getChunkCount()),
100,
updatedAt,
session.getExpiresAt()
));
}
@Override
public void markCancelled(UploadSession session, LocalDateTime updatedAt) {
rewritePhase(session, "cancelled", updatedAt);
}
@Override
public void markFailed(UploadSession session, LocalDateTime updatedAt) {
rewritePhase(session, "failed", updatedAt);
}
@Override
public void markExpired(UploadSession session, LocalDateTime updatedAt) {
rewritePhase(session, "expired", updatedAt);
}
private void rewritePhase(UploadSession session, String phase, LocalDateTime updatedAt) {
UploadSessionRuntimeState current = getState(session.getSessionId()).orElse(new UploadSessionRuntimeState(
phase,
0L,
0,
0,
updatedAt,
session.getExpiresAt()
));
writeState(session, new UploadSessionRuntimeState(
phase,
current.uploadedBytes(),
current.uploadedPartCount(),
current.progressPercent(),
updatedAt,
session.getExpiresAt()
));
}
private void writeState(UploadSession session, UploadSessionRuntimeState state) {
if (session == null || !StringUtils.hasText(session.getSessionId())) {
return;
}
try {
stringRedisTemplate.opsForValue().set(
buildKey(session.getSessionId()),
objectMapper.writeValueAsString(state),
resolveTtl(session.getExpiresAt(), state.phase())
);
} catch (JsonProcessingException ignored) {
}
}
private Duration resolveTtl(LocalDateTime expiresAt, String phase) {
Duration base = Duration.ofSeconds(Math.max(redisProperties.getTtlBufferSeconds(), 60L));
if (expiresAt == null) {
return base;
}
long seconds = Math.max(1L, expiresAt.toEpochSecond(ZoneOffset.UTC) - LocalDateTime.now(ZoneOffset.UTC).toEpochSecond(ZoneOffset.UTC));
Duration sessionWindow = Duration.ofSeconds(seconds + redisProperties.getTtlBufferSeconds());
if ("completed".equals(phase) || "cancelled".equals(phase) || "failed".equals(phase) || "expired".equals(phase)) {
return sessionWindow.compareTo(Duration.ofHours(1)) < 0 ? sessionWindow : Duration.ofHours(1);
}
return sessionWindow;
}
private Integer toProgressPercent(long uploadedBytes, Long totalBytes) {
if (totalBytes == null || totalBytes <= 0) {
return 0;
}
double ratio = Math.min(1.0d, Math.max(0.0d, (double) uploadedBytes / totalBytes));
return (int) Math.round(ratio * 100);
}
private LocalDateTime safeUpdatedAt(UploadSession session) {
return session.getUpdatedAt() == null ? LocalDateTime.now(ZoneOffset.UTC) : session.getUpdatedAt();
}
private String buildKey(String sessionId) {
return redisProperties.getKeyPrefix()
+ ":" + redisProperties.getNamespaces().getUploadState()
+ ":session:" + sessionId.trim();
}
}

View File

@@ -0,0 +1,54 @@
package com.yoyuzh.files.upload;
import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyCapabilities;
import org.springframework.stereotype.Component;
@Component
public class UploadPolicyResolver {
public UploadSessionUploadMode resolveUploadMode(StoragePolicyCapabilities capabilities) {
if (!capabilities.directUpload()) {
return UploadSessionUploadMode.PROXY;
}
if (capabilities.multipartUpload()) {
return UploadSessionUploadMode.DIRECT_MULTIPART;
}
return UploadSessionUploadMode.DIRECT_SINGLE;
}
public long resolveEffectiveMaxUploadSize(long systemMaxFileSize,
User user,
StoragePolicy policy,
StoragePolicyCapabilities capabilities) {
long effectiveMaxUploadSize = Math.min(systemMaxFileSize, user.getMaxUploadSizeBytes());
if (policy.getMaxSizeBytes() > 0) {
effectiveMaxUploadSize = Math.min(effectiveMaxUploadSize, policy.getMaxSizeBytes());
}
if (capabilities.maxObjectSize() > 0) {
effectiveMaxUploadSize = Math.min(effectiveMaxUploadSize, capabilities.maxObjectSize());
}
return effectiveMaxUploadSize;
}
public int calculateChunkCount(long size, long chunkSize) {
if (size <= 0) {
return 1;
}
return (int) Math.ceil((double) size / chunkSize);
}
public long resolveChunkSize(UploadSession session, int partIndex) {
if (partIndex < 0 || partIndex >= session.getChunkCount()) {
throw new BusinessException(ErrorCode.UNKNOWN, "鍒嗙墖搴忓彿涓嶅悎娉?");
}
if (partIndex < session.getChunkCount() - 1) {
return session.getChunkSize();
}
long remaining = session.getSize() - session.getChunkSize() * (session.getChunkCount() - 1L);
return remaining > 0 ? remaining : session.getChunkSize();
}
}

View File

@@ -0,0 +1,13 @@
package com.yoyuzh.files.upload;
import java.time.LocalDateTime;
public record UploadSessionRuntimeState(
String phase,
long uploadedBytes,
int uploadedPartCount,
Integer progressPercent,
LocalDateTime lastUpdatedAt,
LocalDateTime expiresAt
) {
}

View File

@@ -0,0 +1,61 @@
package com.yoyuzh.files.upload;
import java.time.LocalDateTime;
import java.util.Optional;
public interface UploadSessionRuntimeStateService {
Optional<UploadSessionRuntimeState> getState(String sessionId);
void markCreated(UploadSession session);
void markUploading(UploadSession session, long uploadedBytes, int uploadedPartCount, LocalDateTime updatedAt);
void markCompleted(UploadSession session, LocalDateTime updatedAt);
void markCancelled(UploadSession session, LocalDateTime updatedAt);
void markFailed(UploadSession session, LocalDateTime updatedAt);
void markExpired(UploadSession session, LocalDateTime updatedAt);
static UploadSessionRuntimeStateService noOp() {
return NoOpHolder.INSTANCE;
}
final class NoOpHolder {
private static final UploadSessionRuntimeStateService INSTANCE = new UploadSessionRuntimeStateService() {
@Override
public Optional<UploadSessionRuntimeState> getState(String sessionId) {
return Optional.empty();
}
@Override
public void markCreated(UploadSession session) {
}
@Override
public void markUploading(UploadSession session, long uploadedBytes, int uploadedPartCount, LocalDateTime updatedAt) {
}
@Override
public void markCompleted(UploadSession session, LocalDateTime updatedAt) {
}
@Override
public void markCancelled(UploadSession session, LocalDateTime updatedAt) {
}
@Override
public void markFailed(UploadSession session, LocalDateTime updatedAt) {
}
@Override
public void markExpired(UploadSession session, LocalDateTime updatedAt) {
}
};
private NoOpHolder() {
}
}
}

View File

@@ -6,8 +6,10 @@ import com.yoyuzh.auth.User;
import com.yoyuzh.common.BusinessException; import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode; import com.yoyuzh.common.ErrorCode;
import com.yoyuzh.config.FileStorageProperties; import com.yoyuzh.config.FileStorageProperties;
import com.yoyuzh.files.core.FileUploadRulesService;
import com.yoyuzh.files.core.FileService; import com.yoyuzh.files.core.FileService;
import com.yoyuzh.files.core.StoredFileRepository; import com.yoyuzh.files.core.StoredFileRepository;
import com.yoyuzh.files.core.WorkspaceNodeRulesService;
import com.yoyuzh.files.policy.StoragePolicy; import com.yoyuzh.files.policy.StoragePolicy;
import com.yoyuzh.files.policy.StoragePolicyCapabilities; import com.yoyuzh.files.policy.StoragePolicyCapabilities;
import com.yoyuzh.files.policy.StoragePolicyService; import com.yoyuzh.files.policy.StoragePolicyService;
@@ -26,6 +28,7 @@ import java.time.LocalDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.UUID; import java.util.UUID;
@Service @Service
@@ -42,13 +45,17 @@ public class UploadSessionService {
}; };
private final UploadSessionRepository uploadSessionRepository; private final UploadSessionRepository uploadSessionRepository;
private final StoredFileRepository storedFileRepository;
private final FileService fileService; private final FileService fileService;
private final FileContentStorage fileContentStorage; private final FileContentStorage fileContentStorage;
private final StoragePolicyService storagePolicyService; private final StoragePolicyService storagePolicyService;
private final ObjectMapper objectMapper = new ObjectMapper(); private final ObjectMapper objectMapper = new ObjectMapper();
private final long maxFileSize; private final UploadPolicyResolver uploadPolicyResolver;
private final UploadSessionStateMachine uploadSessionStateMachine;
private final WorkspaceNodeRulesService workspaceNodeRulesService;
private final FileUploadRulesService fileUploadRulesService;
private final Clock clock; private final Clock clock;
@Autowired(required = false)
private UploadSessionRuntimeStateService uploadSessionRuntimeStateService = UploadSessionRuntimeStateService.noOp();
@Autowired @Autowired
public UploadSessionService(UploadSessionRepository uploadSessionRepository, public UploadSessionService(UploadSessionRepository uploadSessionRepository,
@@ -56,8 +63,20 @@ public class UploadSessionService {
FileService fileService, FileService fileService,
FileContentStorage fileContentStorage, FileContentStorage fileContentStorage,
StoragePolicyService storagePolicyService, StoragePolicyService storagePolicyService,
FileStorageProperties properties) { FileStorageProperties properties,
this(uploadSessionRepository, storedFileRepository, fileService, fileContentStorage, storagePolicyService, properties, Clock.systemUTC()); UploadPolicyResolver uploadPolicyResolver,
UploadSessionStateMachine uploadSessionStateMachine) {
this(
uploadSessionRepository,
storedFileRepository,
fileService,
fileContentStorage,
storagePolicyService,
properties,
Clock.systemUTC(),
uploadPolicyResolver,
uploadSessionStateMachine
);
} }
UploadSessionService(UploadSessionRepository uploadSessionRepository, UploadSessionService(UploadSessionRepository uploadSessionRepository,
@@ -67,23 +86,52 @@ public class UploadSessionService {
StoragePolicyService storagePolicyService, StoragePolicyService storagePolicyService,
FileStorageProperties properties, FileStorageProperties properties,
Clock clock) { Clock clock) {
this(
uploadSessionRepository,
storedFileRepository,
fileService,
fileContentStorage,
storagePolicyService,
properties,
clock,
new UploadPolicyResolver(),
new UploadSessionStateMachine()
);
}
UploadSessionService(UploadSessionRepository uploadSessionRepository,
StoredFileRepository storedFileRepository,
FileService fileService,
FileContentStorage fileContentStorage,
StoragePolicyService storagePolicyService,
FileStorageProperties properties,
Clock clock,
UploadPolicyResolver uploadPolicyResolver,
UploadSessionStateMachine uploadSessionStateMachine) {
this.uploadSessionRepository = uploadSessionRepository; this.uploadSessionRepository = uploadSessionRepository;
this.storedFileRepository = storedFileRepository;
this.fileService = fileService; this.fileService = fileService;
this.fileContentStorage = fileContentStorage; this.fileContentStorage = fileContentStorage;
this.storagePolicyService = storagePolicyService; this.storagePolicyService = storagePolicyService;
this.maxFileSize = properties.getMaxFileSize();
this.clock = clock; this.clock = clock;
this.uploadPolicyResolver = uploadPolicyResolver;
this.uploadSessionStateMachine = uploadSessionStateMachine;
this.workspaceNodeRulesService = new WorkspaceNodeRulesService(storedFileRepository, fileContentStorage);
this.fileUploadRulesService = new FileUploadRulesService(
storedFileRepository,
storagePolicyService,
workspaceNodeRulesService,
properties.getMaxFileSize()
);
} }
@Transactional @Transactional
public UploadSession createSession(User user, UploadSessionCreateCommand command) { public UploadSession createSession(User user, UploadSessionCreateCommand command) {
String normalizedPath = normalizeDirectoryPath(command.path()); String normalizedPath = workspaceNodeRulesService.normalizeDirectoryPath(command.path());
String filename = normalizeLeafName(command.filename()); String filename = workspaceNodeRulesService.normalizeLeafName(command.filename());
StoragePolicy policy = storagePolicyService.ensureDefaultPolicy(); StoragePolicy policy = storagePolicyService.ensureDefaultPolicy();
StoragePolicyCapabilities capabilities = storagePolicyService.readCapabilities(policy); StoragePolicyCapabilities capabilities = storagePolicyService.readCapabilities(policy);
validateTarget(user, normalizedPath, filename, command.size(), policy, capabilities); validateTarget(user, normalizedPath, filename, command.size());
UploadSessionUploadMode uploadMode = resolveUploadMode(capabilities); UploadSessionUploadMode uploadMode = uploadPolicyResolver.resolveUploadMode(capabilities);
UploadSession session = new UploadSession(); UploadSession session = new UploadSession();
session.setSessionId(UUID.randomUUID().toString()); session.setSessionId(UUID.randomUUID().toString());
@@ -96,44 +144,51 @@ public class UploadSessionService {
session.setStoragePolicyId(policy.getId()); session.setStoragePolicyId(policy.getId());
session.setChunkSize(DEFAULT_CHUNK_SIZE); session.setChunkSize(DEFAULT_CHUNK_SIZE);
session.setChunkCount(uploadMode == UploadSessionUploadMode.DIRECT_MULTIPART session.setChunkCount(uploadMode == UploadSessionUploadMode.DIRECT_MULTIPART
? calculateChunkCount(command.size(), DEFAULT_CHUNK_SIZE) ? uploadPolicyResolver.calculateChunkCount(command.size(), DEFAULT_CHUNK_SIZE)
: 1); : 1);
session.setUploadedPartsJson("[]"); session.setUploadedPartsJson("[]");
session.setStatus(UploadSessionStatus.CREATED); session.setStatus(UploadSessionStatus.CREATED);
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); LocalDateTime now = now();
session.setCreatedAt(now); session.setCreatedAt(now);
session.setUpdatedAt(now); session.setUpdatedAt(now);
session.setExpiresAt(now.plusHours(SESSION_TTL_HOURS)); session.setExpiresAt(now.plusHours(SESSION_TTL_HOURS));
if (uploadMode == UploadSessionUploadMode.DIRECT_MULTIPART) { if (uploadMode == UploadSessionUploadMode.DIRECT_MULTIPART) {
session.setMultipartUploadId(fileContentStorage.createMultipartUpload(session.getObjectKey(), session.getContentType())); session.setMultipartUploadId(fileContentStorage.createMultipartUpload(session.getObjectKey(), session.getContentType()));
} }
return uploadSessionRepository.save(session); UploadSession savedSession = uploadSessionRepository.save(session);
uploadSessionRuntimeStateService.markCreated(savedSession);
return savedSession;
} }
@Transactional(readOnly = true) @Transactional(readOnly = true)
public UploadSession getOwnedSession(User user, String sessionId) { public UploadSession getOwnedSession(User user, String sessionId) {
return uploadSessionRepository.findBySessionIdAndUserId(sessionId, user.getId()) return uploadSessionRepository.findBySessionIdAndUserId(sessionId, user.getId())
.orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "上传会话不存在")); .orElseThrow(() -> new BusinessException(ErrorCode.FILE_NOT_FOUND, "upload session not found"));
}
@Transactional(readOnly = true)
public Optional<UploadSessionRuntimeState> getRuntimeState(String sessionId) {
return uploadSessionRuntimeStateService.getState(sessionId);
} }
@Transactional @Transactional
public UploadSession cancelOwnedSession(User user, String sessionId) { public UploadSession cancelOwnedSession(User user, String sessionId) {
UploadSession session = getOwnedSession(user, sessionId); UploadSession session = getOwnedSession(user, sessionId);
if (session.getStatus() == UploadSessionStatus.COMPLETED) { if (session.getStatus() == UploadSessionStatus.COMPLETED) {
throw new BusinessException(ErrorCode.UNKNOWN, "已完成的上传会话不能取消"); throw new BusinessException(ErrorCode.UNKNOWN, "completed upload session cannot be cancelled");
} }
session.setStatus(UploadSessionStatus.CANCELLED); uploadSessionStateMachine.markCancelled(session, now());
session.setUpdatedAt(LocalDateTime.ofInstant(clock.instant(), clock.getZone())); UploadSession savedSession = uploadSessionRepository.save(session);
return uploadSessionRepository.save(session); uploadSessionRuntimeStateService.markCancelled(savedSession, savedSession.getUpdatedAt());
return savedSession;
} }
@Transactional(readOnly = true) @Transactional(readOnly = true)
public PreparedUpload prepareOwnedUpload(User user, String sessionId) { public PreparedUpload prepareOwnedUpload(User user, String sessionId) {
UploadSession session = getOwnedSession(user, sessionId); UploadSession session = getOwnedSession(user, sessionId);
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); ensureSessionCanReceiveContent(session, now());
ensureSessionCanReceiveContent(session, now);
if (resolveUploadMode(session) != UploadSessionUploadMode.DIRECT_SINGLE) { if (resolveUploadMode(session) != UploadSessionUploadMode.DIRECT_SINGLE) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话未启用单请求直传"); throw new BusinessException(ErrorCode.UNKNOWN, "upload session does not support direct single upload");
} }
return fileContentStorage.prepareBlobUpload( return fileContentStorage.prepareBlobUpload(
session.getTargetPath(), session.getTargetPath(),
@@ -147,21 +202,20 @@ public class UploadSessionService {
@Transactional(readOnly = true) @Transactional(readOnly = true)
public PreparedUpload prepareOwnedPartUpload(User user, String sessionId, int partIndex) { public PreparedUpload prepareOwnedPartUpload(User user, String sessionId, int partIndex) {
UploadSession session = getOwnedSession(user, sessionId); UploadSession session = getOwnedSession(user, sessionId);
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); ensureSessionCanReceivePart(session, now());
ensureSessionCanReceivePart(session, now);
if (resolveUploadMode(session) != UploadSessionUploadMode.DIRECT_MULTIPART if (resolveUploadMode(session) != UploadSessionUploadMode.DIRECT_MULTIPART
|| !StringUtils.hasText(session.getMultipartUploadId())) { || !StringUtils.hasText(session.getMultipartUploadId())) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话未启用 multipart"); throw new BusinessException(ErrorCode.UNKNOWN, "upload session does not support multipart upload");
} }
if (partIndex < 0 || partIndex >= session.getChunkCount()) { if (partIndex < 0 || partIndex >= session.getChunkCount()) {
throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法"); throw new BusinessException(ErrorCode.UNKNOWN, "invalid part index");
} }
return fileContentStorage.prepareMultipartPartUpload( return fileContentStorage.prepareMultipartPartUpload(
session.getObjectKey(), session.getObjectKey(),
session.getMultipartUploadId(), session.getMultipartUploadId(),
partIndex + 1, partIndex + 1,
session.getContentType(), session.getContentType(),
resolveChunkSize(session, partIndex) uploadPolicyResolver.resolveChunkSize(session, partIndex)
); );
} }
@@ -171,19 +225,19 @@ public class UploadSessionService {
int partIndex, int partIndex,
UploadSessionPartCommand command) { UploadSessionPartCommand command) {
UploadSession session = getOwnedSession(user, sessionId); UploadSession session = getOwnedSession(user, sessionId);
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); LocalDateTime now = now();
ensureSessionCanReceivePart(session, now); ensureSessionCanReceivePart(session, now);
if (resolveUploadMode(session) != UploadSessionUploadMode.DIRECT_MULTIPART) { if (resolveUploadMode(session) != UploadSessionUploadMode.DIRECT_MULTIPART) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话未启用 multipart"); throw new BusinessException(ErrorCode.UNKNOWN, "upload session does not support multipart upload");
} }
if (partIndex < 0 || partIndex >= session.getChunkCount()) { if (partIndex < 0 || partIndex >= session.getChunkCount()) {
throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法"); throw new BusinessException(ErrorCode.UNKNOWN, "invalid part index");
} }
if (!StringUtils.hasText(command.etag())) { if (!StringUtils.hasText(command.etag())) {
throw new BusinessException(ErrorCode.UNKNOWN, "分片标识不能为空"); throw new BusinessException(ErrorCode.UNKNOWN, "part etag is required");
} }
if (command.size() < 0) { if (command.size() < 0) {
throw new BusinessException(ErrorCode.UNKNOWN, "分片大小不合法"); throw new BusinessException(ErrorCode.UNKNOWN, "invalid part size");
} }
List<UploadedPart> uploadedParts = new ArrayList<>(readUploadedParts(session)); List<UploadedPart> uploadedParts = new ArrayList<>(readUploadedParts(session));
@@ -192,33 +246,42 @@ public class UploadSessionService {
uploadedParts.sort(Comparator.comparingInt(UploadedPart::partIndex)); uploadedParts.sort(Comparator.comparingInt(UploadedPart::partIndex));
session.setUploadedPartsJson(writeUploadedParts(uploadedParts)); session.setUploadedPartsJson(writeUploadedParts(uploadedParts));
if (session.getStatus() == UploadSessionStatus.CREATED) { uploadSessionStateMachine.markUploading(session, now);
session.setStatus(UploadSessionStatus.UPLOADING); UploadSession savedSession = uploadSessionRepository.save(session);
} long uploadedBytes = uploadedParts.stream().mapToLong(UploadedPart::size).sum();
session.setUpdatedAt(now); uploadSessionRuntimeStateService.markUploading(
return uploadSessionRepository.save(session); savedSession,
uploadedBytes,
uploadedParts.size(),
savedSession.getUpdatedAt()
);
return savedSession;
} }
@Transactional @Transactional
public UploadSession uploadOwnedContent(User user, String sessionId, MultipartFile file) { public UploadSession uploadOwnedContent(User user, String sessionId, MultipartFile file) {
UploadSession session = getOwnedSession(user, sessionId); UploadSession session = getOwnedSession(user, sessionId);
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); LocalDateTime now = now();
ensureSessionCanReceiveContent(session, now); ensureSessionCanReceiveContent(session, now);
if (resolveUploadMode(session) != UploadSessionUploadMode.PROXY) { if (resolveUploadMode(session) != UploadSessionUploadMode.PROXY) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话未启用代理上传"); throw new BusinessException(ErrorCode.UNKNOWN, "upload session does not support proxy upload");
} }
if (file == null || file.isEmpty()) { if (file == null || file.isEmpty()) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传内容不能为空"); throw new BusinessException(ErrorCode.UNKNOWN, "upload content is required");
} }
if (file.getSize() != session.getSize()) { if (file.getSize() != session.getSize()) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传内容大小与会话不一致"); throw new BusinessException(ErrorCode.UNKNOWN, "upload size does not match session");
} }
fileContentStorage.uploadBlob(session.getObjectKey(), file); fileContentStorage.uploadBlob(session.getObjectKey(), file);
if (session.getStatus() == UploadSessionStatus.CREATED) { uploadSessionStateMachine.markUploading(session, now);
session.setStatus(UploadSessionStatus.UPLOADING); UploadSession savedSession = uploadSessionRepository.save(session);
} uploadSessionRuntimeStateService.markUploading(
session.setUpdatedAt(now); savedSession,
return uploadSessionRepository.save(session); savedSession.getSize(),
1,
savedSession.getUpdatedAt()
);
return savedSession;
} }
@Transactional @Transactional
@@ -228,19 +291,24 @@ public class UploadSessionService {
return session; return session;
} }
if (session.getStatus() == UploadSessionStatus.CANCELLED || session.getStatus() == UploadSessionStatus.FAILED) { if (session.getStatus() == UploadSessionStatus.CANCELLED || session.getStatus() == UploadSessionStatus.FAILED) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话不能完成"); throw new BusinessException(ErrorCode.UNKNOWN, "upload session cannot be completed");
} }
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); LocalDateTime now = now();
if (session.getExpiresAt().isBefore(now)) { if (session.getExpiresAt().isBefore(now)) {
session.setStatus(UploadSessionStatus.EXPIRED); uploadSessionStateMachine.markExpired(session, now);
session.setUpdatedAt(now); UploadSession expiredSession = uploadSessionRepository.save(session);
uploadSessionRepository.save(session); uploadSessionRuntimeStateService.markExpired(expiredSession, expiredSession.getUpdatedAt());
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话已过期"); throw new BusinessException(ErrorCode.UNKNOWN, "upload session has expired");
} }
session.setStatus(UploadSessionStatus.COMPLETING); uploadSessionStateMachine.markCompleting(session, now);
session.setUpdatedAt(now); UploadSession completingSession = uploadSessionRepository.save(session);
uploadSessionRepository.save(session); uploadSessionRuntimeStateService.markUploading(
completingSession,
completingSession.getSize() == null ? 0L : completingSession.getSize(),
Math.max(1, completingSession.getChunkCount() == null ? 1 : completingSession.getChunkCount()),
completingSession.getUpdatedAt()
);
try { try {
if (resolveUploadMode(session) == UploadSessionUploadMode.DIRECT_MULTIPART if (resolveUploadMode(session) == UploadSessionUploadMode.DIRECT_MULTIPART
@@ -258,13 +326,14 @@ public class UploadSessionService {
session.getContentType(), session.getContentType(),
session.getSize() session.getSize()
)); ));
session.setStatus(UploadSessionStatus.COMPLETED); uploadSessionStateMachine.markCompleted(session, now);
session.setUpdatedAt(now); UploadSession completedSession = uploadSessionRepository.save(session);
return uploadSessionRepository.save(session); uploadSessionRuntimeStateService.markCompleted(completedSession, completedSession.getUpdatedAt());
return completedSession;
} catch (RuntimeException ex) { } catch (RuntimeException ex) {
session.setStatus(UploadSessionStatus.FAILED); uploadSessionStateMachine.markFailed(session, now);
session.setUpdatedAt(now); UploadSession failedSession = uploadSessionRepository.save(session);
uploadSessionRepository.save(session); uploadSessionRuntimeStateService.markFailed(failedSession, failedSession.getUpdatedAt());
throw ex; throw ex;
} }
} }
@@ -272,7 +341,7 @@ public class UploadSessionService {
@Scheduled(fixedDelay = 60 * 60 * 1000L) @Scheduled(fixedDelay = 60 * 60 * 1000L)
@Transactional @Transactional
public int pruneExpiredSessions() { public int pruneExpiredSessions() {
LocalDateTime now = LocalDateTime.ofInstant(clock.instant(), clock.getZone()); LocalDateTime now = now();
List<UploadSession> expiredSessions = uploadSessionRepository.findByStatusInAndExpiresAtBefore( List<UploadSession> expiredSessions = uploadSessionRepository.findByStatusInAndExpiresAtBefore(
EXPIRABLE_STATUSES, EXPIRABLE_STATUSES,
now now
@@ -287,8 +356,8 @@ public class UploadSessionService {
} catch (RuntimeException ignored) { } catch (RuntimeException ignored) {
// Expiration is authoritative in the database even if remote object cleanup fails. // Expiration is authoritative in the database even if remote object cleanup fails.
} }
session.setStatus(UploadSessionStatus.EXPIRED); uploadSessionStateMachine.markExpired(session, now);
session.setUpdatedAt(now); uploadSessionRuntimeStateService.markExpired(session, session.getUpdatedAt());
} }
if (!expiredSessions.isEmpty()) { if (!expiredSessions.isEmpty()) {
uploadSessionRepository.saveAll(expiredSessions); uploadSessionRepository.saveAll(expiredSessions);
@@ -304,64 +373,44 @@ public class UploadSessionService {
return UploadSessionUploadMode.PROXY; return UploadSessionUploadMode.PROXY;
} }
StoragePolicy policy = storagePolicyService.getRequiredPolicy(session.getStoragePolicyId()); StoragePolicy policy = storagePolicyService.getRequiredPolicy(session.getStoragePolicyId());
return resolveUploadMode(storagePolicyService.readCapabilities(policy)); return uploadPolicyResolver.resolveUploadMode(storagePolicyService.readCapabilities(policy));
}
private UploadSessionUploadMode resolveUploadMode(StoragePolicyCapabilities capabilities) {
if (!capabilities.directUpload()) {
return UploadSessionUploadMode.PROXY;
}
if (capabilities.multipartUpload()) {
return UploadSessionUploadMode.DIRECT_MULTIPART;
}
return UploadSessionUploadMode.DIRECT_SINGLE;
} }
private void validateTarget(User user, private void validateTarget(User user,
String normalizedPath, String normalizedPath,
String filename, String filename,
long size, long size) {
StoragePolicy policy, fileUploadRulesService.validateUpload(user, normalizedPath, filename, size);
StoragePolicyCapabilities capabilities) {
long effectiveMaxUploadSize = Math.min(maxFileSize, user.getMaxUploadSizeBytes());
if (policy.getMaxSizeBytes() > 0) {
effectiveMaxUploadSize = Math.min(effectiveMaxUploadSize, policy.getMaxSizeBytes());
}
if (capabilities.maxObjectSize() > 0) {
effectiveMaxUploadSize = Math.min(effectiveMaxUploadSize, capabilities.maxObjectSize());
}
if (size > effectiveMaxUploadSize) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件大小超出限制");
}
if (storedFileRepository.existsByUserIdAndPathAndFilename(user.getId(), normalizedPath, filename)) {
throw new BusinessException(ErrorCode.UNKNOWN, "同目录下文件已存在");
}
long usedBytes = storedFileRepository.sumFileSizeByUserId(user.getId());
if (user.getStorageQuotaBytes() >= 0 && usedBytes + size > user.getStorageQuotaBytes()) {
throw new BusinessException(ErrorCode.UNKNOWN, "存储空间不足");
}
} }
private void ensureSessionCanReceiveContent(UploadSession session, LocalDateTime now) { private void ensureSessionCanReceiveContent(UploadSession session, LocalDateTime now) {
ensureSessionCanReceivePart(session, now); try {
if (session.getStatus() == UploadSessionStatus.UPLOADING && StringUtils.hasText(session.getMultipartUploadId())) { uploadSessionStateMachine.ensureCanReceiveContent(
throw new BusinessException(ErrorCode.UNKNOWN, "multipart 上传会话不能走整体内容上传"); session,
now,
StringUtils.hasText(session.getMultipartUploadId())
);
} catch (BusinessException ex) {
markRuntimeExpiredIfNeeded(session);
throw ex;
} }
} }
private void ensureSessionCanReceivePart(UploadSession session, LocalDateTime now) { private void ensureSessionCanReceivePart(UploadSession session, LocalDateTime now) {
if (session.getStatus() == UploadSessionStatus.CANCELLED try {
|| session.getStatus() == UploadSessionStatus.FAILED uploadSessionStateMachine.ensureCanReceivePart(session, now);
|| session.getStatus() == UploadSessionStatus.COMPLETING } catch (BusinessException ex) {
|| session.getStatus() == UploadSessionStatus.COMPLETED) { markRuntimeExpiredIfNeeded(session);
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话不能继续上传分片"); throw ex;
} }
if (session.getExpiresAt().isBefore(now)) {
session.setStatus(UploadSessionStatus.EXPIRED);
session.setUpdatedAt(now);
uploadSessionRepository.save(session);
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话已过期");
} }
private void markRuntimeExpiredIfNeeded(UploadSession session) {
if (session.getStatus() != UploadSessionStatus.EXPIRED) {
return;
}
UploadSession expiredSession = uploadSessionRepository.save(session);
uploadSessionRuntimeStateService.markExpired(expiredSession, expiredSession.getUpdatedAt());
} }
private List<UploadedPart> readUploadedParts(UploadSession session) { private List<UploadedPart> readUploadedParts(UploadSession session) {
@@ -371,7 +420,7 @@ public class UploadSessionService {
try { try {
return objectMapper.readValue(session.getUploadedPartsJson(), UPLOADED_PARTS_TYPE); return objectMapper.readValue(session.getUploadedPartsJson(), UPLOADED_PARTS_TYPE);
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话分片状态不合法"); throw new BusinessException(ErrorCode.UNKNOWN, "invalid uploaded part state");
} }
} }
@@ -379,7 +428,7 @@ public class UploadSessionService {
try { try {
return objectMapper.writeValueAsString(uploadedParts); return objectMapper.writeValueAsString(uploadedParts);
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传会话分片状态写入失败"); throw new BusinessException(ErrorCode.UNKNOWN, "failed to write uploaded part state");
} }
} }
@@ -388,18 +437,18 @@ public class UploadSessionService {
.sorted(Comparator.comparingInt(UploadedPart::partIndex)) .sorted(Comparator.comparingInt(UploadedPart::partIndex))
.toList(); .toList();
if (uploadedParts.size() != session.getChunkCount()) { if (uploadedParts.size() != session.getChunkCount()) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片不完整"); throw new BusinessException(ErrorCode.UNKNOWN, "multipart upload is incomplete");
} }
for (int expectedIndex = 0; expectedIndex < session.getChunkCount(); expectedIndex++) { for (int expectedIndex = 0; expectedIndex < session.getChunkCount(); expectedIndex++) {
UploadedPart part = uploadedParts.get(expectedIndex); UploadedPart part = uploadedParts.get(expectedIndex);
if (part.partIndex() != expectedIndex) { if (part.partIndex() != expectedIndex) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片不完整"); throw new BusinessException(ErrorCode.UNKNOWN, "multipart upload is incomplete");
} }
if (!StringUtils.hasText(part.etag())) { if (!StringUtils.hasText(part.etag())) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片标识缺失"); throw new BusinessException(ErrorCode.UNKNOWN, "missing part etag");
} }
if (part.size() <= 0 || part.size() > resolveChunkSize(session, expectedIndex)) { if (part.size() <= 0 || part.size() > uploadPolicyResolver.resolveChunkSize(session, expectedIndex)) {
throw new BusinessException(ErrorCode.UNKNOWN, "上传分片大小不合法"); throw new BusinessException(ErrorCode.UNKNOWN, "invalid part size");
} }
} }
return uploadedParts.stream() return uploadedParts.stream()
@@ -407,53 +456,15 @@ public class UploadSessionService {
.toList(); .toList();
} }
private long resolveChunkSize(UploadSession session, int partIndex) {
if (partIndex < 0 || partIndex >= session.getChunkCount()) {
throw new BusinessException(ErrorCode.UNKNOWN, "分片序号不合法");
}
if (partIndex < session.getChunkCount() - 1) {
return session.getChunkSize();
}
long remaining = session.getSize() - session.getChunkSize() * (session.getChunkCount() - 1L);
return remaining > 0 ? remaining : session.getChunkSize();
}
private record UploadedPart(int partIndex, String etag, long size, String uploadedAt) { private record UploadedPart(int partIndex, String etag, long size, String uploadedAt) {
} }
private int calculateChunkCount(long size, long chunkSize) {
if (size <= 0) {
return 1;
}
return (int) Math.ceil((double) size / chunkSize);
}
private String createBlobObjectKey() { private String createBlobObjectKey() {
return "blobs/" + UUID.randomUUID(); return "blobs/" + UUID.randomUUID();
} }
private String normalizeDirectoryPath(String path) { private LocalDateTime now() {
String cleaned = StringUtils.cleanPath(path == null ? "/" : path.trim().replace("\\", "/")); return LocalDateTime.ofInstant(clock.instant(), clock.getZone());
if (!cleaned.startsWith("/")) {
cleaned = "/" + cleaned;
}
while (cleaned.length() > 1 && cleaned.endsWith("/")) {
cleaned = cleaned.substring(0, cleaned.length() - 1);
}
if (!StringUtils.hasText(cleaned) || cleaned.contains("..")) {
throw new BusinessException(ErrorCode.UNKNOWN, "路径不合法");
}
return cleaned;
} }
private String normalizeLeafName(String filename) {
String cleaned = StringUtils.cleanPath(filename == null ? "" : filename).trim();
if (!StringUtils.hasText(cleaned)) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件名不能为空");
}
if (cleaned.contains("/") || cleaned.contains("\\") || cleaned.contains("..")) {
throw new BusinessException(ErrorCode.UNKNOWN, "文件名不合法");
}
return cleaned;
}
} }

View File

@@ -0,0 +1,63 @@
package com.yoyuzh.files.upload;
import com.yoyuzh.common.BusinessException;
import com.yoyuzh.common.ErrorCode;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
@Component
public class UploadSessionStateMachine {
public void ensureCanReceivePart(UploadSession session, LocalDateTime now) {
if (session.getStatus() == UploadSessionStatus.CANCELLED
|| session.getStatus() == UploadSessionStatus.FAILED
|| session.getStatus() == UploadSessionStatus.COMPLETING
|| session.getStatus() == UploadSessionStatus.COMPLETED) {
throw new BusinessException(ErrorCode.UNKNOWN, "涓婁紶浼氳瘽涓嶈兘缁х画涓婁紶鍒嗙墖");
}
if (session.getExpiresAt().isBefore(now)) {
markExpired(session, now);
throw new BusinessException(ErrorCode.UNKNOWN, "涓婁紶浼氳瘽宸茶繃鏈?");
}
}
public void ensureCanReceiveContent(UploadSession session, LocalDateTime now, boolean multipartUpload) {
ensureCanReceivePart(session, now);
if (session.getStatus() == UploadSessionStatus.UPLOADING && multipartUpload) {
throw new BusinessException(ErrorCode.UNKNOWN, "multipart 涓婁紶浼氳瘽涓嶈兘璧版暣浣撳唴瀹逛笂浼?");
}
}
public void markUploading(UploadSession session, LocalDateTime now) {
if (session.getStatus() == UploadSessionStatus.CREATED) {
session.setStatus(UploadSessionStatus.UPLOADING);
}
session.setUpdatedAt(now);
}
public void markCompleting(UploadSession session, LocalDateTime now) {
session.setStatus(UploadSessionStatus.COMPLETING);
session.setUpdatedAt(now);
}
public void markCompleted(UploadSession session, LocalDateTime now) {
session.setStatus(UploadSessionStatus.COMPLETED);
session.setUpdatedAt(now);
}
public void markFailed(UploadSession session, LocalDateTime now) {
session.setStatus(UploadSessionStatus.FAILED);
session.setUpdatedAt(now);
}
public void markCancelled(UploadSession session, LocalDateTime now) {
session.setStatus(UploadSessionStatus.CANCELLED);
session.setUpdatedAt(now);
}
public void markExpired(UploadSession session, LocalDateTime now) {
session.setStatus(UploadSessionStatus.EXPIRED);
session.setUpdatedAt(now);
}
}

Some files were not shown because too many files have changed in this diff Show More