yoyuz/my_site
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Eliminate admin N+1 queries and lazy-load app routes

Browse Source
This commit is contained in:
yoyuzh
2026-04-12 00:48:23 +08:00
parent 30a9bbc1e7
commit 9af2d38e37
9 changed files with 290 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
backend/src/main/java/com/yoyuzh/admin/AdminInspectionQueryService.java
View File

@@ -8,6 +8,7 @@ import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.core.FileEntity;
import com.yoyuzh.files.core.FileEntityRepository;
import com.yoyuzh.files.core.FileEntityType;
import com.yoyuzh.files.core.FileBlob;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileEntityRepository;
import com.yoyuzh.files.core.StoredFileRepository;
@@ -23,7 +24,12 @@ import org.springframework.util.StringUtils;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@@ -82,8 +88,15 @@ public class AdminInspectionQueryService {
entityType,
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
List<AdminFileBlobResponse> items = result.getContent().stream()
.map(this::toFileBlobResponse)
List<FileEntity> entities = result.getContent();
Map<String, FileBlob> blobsByObjectKey = loadBlobsByObjectKey(entities);
Map<Long, StoredFileEntityRepository.FileEntityLinkStatsProjection> linkStatsByEntityId = loadLinkStatsByEntityId(entities);
List<AdminFileBlobResponse> items = entities.stream()
.map(entity -> toFileBlobResponse(
entity,
blobsByObjectKey.get(entity.getObjectKey()),
linkStatsByEntityId.get(entity.getId())
))
.toList();
return new PageResponse<>(items, result.getTotalElements(), page, size);
}
@@ -126,10 +139,17 @@ public class AdminInspectionQueryService {
);
}
private AdminFileBlobResponse toFileBlobResponse(FileEntity entity) {
var blob = fileBlobRepository.findByObjectKey(entity.getObjectKey()).orElse(null);
long linkedStoredFileCount = storedFileEntityRepository.countByFileEntityId(entity.getId());
long linkedOwnerCount = storedFileEntityRepository.countDistinctOwnersByFileEntityId(entity.getId());
private AdminFileBlobResponse toFileBlobResponse(FileEntity entity,
FileBlob blob,
StoredFileEntityRepository.FileEntityLinkStatsProjection linkStats) {
long linkedStoredFileCount = linkStats == null || linkStats.getLinkedStoredFileCount() == null
? 0L
: linkStats.getLinkedStoredFileCount();
long linkedOwnerCount = linkStats == null || linkStats.getLinkedOwnerCount() == null
? 0L
: linkStats.getLinkedOwnerCount();
String sampleOwnerUsername = linkStats == null ? null : linkStats.getSampleOwnerUsername();
String sampleOwnerEmail = linkStats == null ? null : linkStats.getSampleOwnerEmail();
return new AdminFileBlobResponse(
entity.getId(),
blob == null ? null : blob.getId(),
@@ -141,8 +161,8 @@ public class AdminInspectionQueryService {
entity.getReferenceCount(),
linkedStoredFileCount,
linkedOwnerCount,
storedFileEntityRepository.findSampleOwnerUsernameByFileEntityId(entity.getId()),
storedFileEntityRepository.findSampleOwnerEmailByFileEntityId(entity.getId()),
sampleOwnerUsername,
sampleOwnerEmail,
entity.getCreatedBy() == null ? null : entity.getCreatedBy().getId(),
entity.getCreatedBy() == null ? null : entity.getCreatedBy().getUsername(),
entity.getCreatedAt(),
@@ -153,6 +173,37 @@ public class AdminInspectionQueryService {
);
}
private Map<String, FileBlob> loadBlobsByObjectKey(List<FileEntity> entities) {
Set<String> objectKeys = entities.stream()
.map(FileEntity::getObjectKey)
.filter(StringUtils::hasText)
.collect(Collectors.toSet());
if (objectKeys.isEmpty()) {
return Map.of();
}
return fileBlobRepository.findAllByObjectKeyIn(objectKeys).stream()
.collect(Collectors.toMap(
FileBlob::getObjectKey,
Function.identity(),
(left, right) -> left
));
}
private Map<Long, StoredFileEntityRepository.FileEntityLinkStatsProjection> loadLinkStatsByEntityId(List<FileEntity> entities) {
Set<Long> entityIds = entities.stream()
.map(FileEntity::getId)
.filter(id -> id != null)
.collect(Collectors.toSet());
if (entityIds.isEmpty()) {
return Collections.emptyMap();
}
return storedFileEntityRepository.findAdminLinkStatsByFileEntityIds(entityIds).stream()
.collect(Collectors.toMap(
StoredFileEntityRepository.FileEntityLinkStatsProjection::getFileEntityId,
Function.identity()
));
}
private AdminShareResponse toAdminShareResponse(FileShareLink shareLink) {
StoredFile file = shareLink.getFile();
User owner = shareLink.getOwner();

29
backend/src/main/java/com/yoyuzh/admin/AdminUserGovernanceService.java
View File

@@ -21,7 +21,10 @@ import org.springframework.transaction.annotation.Transactional;
import java.security.SecureRandom;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@@ -41,8 +44,12 @@ public class AdminUserGovernanceService {
normalizeQuery(query),
PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "createdAt"))
);
List<User> users = result.getContent();
Map<Long, Long> usedStorageByUserId = loadUsedStorageByUserIds(users);
return new PageResponse<>(
result.getContent().stream().map(this::toUserResponse).toList(),
users.stream()
.map(user -> toUserResponse(user, usedStorageByUserId.getOrDefault(user.getId(), 0L)))
.toList(),
result.getTotalElements(),
page,
size
@@ -150,7 +157,10 @@ public class AdminUserGovernanceService {
}
private AdminUserResponse toUserResponse(User user) {
long usedStorageBytes = storedFileRepository.sumFileSizeByUserId(user.getId());
return toUserResponse(user, storedFileRepository.sumFileSizeByUserId(user.getId()));
}
private AdminUserResponse toUserResponse(User user, long usedStorageBytes) {
return new AdminUserResponse(
user.getId(),
user.getUsername(),
@@ -165,6 +175,21 @@ public class AdminUserGovernanceService {
);
}
private Map<Long, Long> loadUsedStorageByUserIds(List<User> users) {
Set<Long> userIds = users.stream()
.map(User::getId)
.filter(id -> id != null)
.collect(Collectors.toSet());
if (userIds.isEmpty()) {
return Map.of();
}
return storedFileRepository.sumFileSizeByUserIds(userIds).stream()
.collect(Collectors.toMap(
StoredFileRepository.UserStorageUsageProjection::getUserId,
projection -> projection.getUsedStorageBytes() == null ? 0L : projection.getUsedStorageBytes()
));
}
private User getRequiredUser(Long userId) {
return userRepository.findById(userId)
.orElseThrow(() -> new BusinessException(ErrorCode.UNKNOWN, "user not found"));

4
backend/src/main/java/com/yoyuzh/files/core/FileBlobRepository.java
View File

@@ -3,12 +3,16 @@ package com.yoyuzh.files.core;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
public interface FileBlobRepository extends JpaRepository<FileBlob, Long> {
Optional<FileBlob> findByObjectKey(String objectKey);
List<FileBlob> findAllByObjectKeyIn(Collection<String> objectKeys);
@Query("""
select coalesce(sum(b.size), 0)
from FileBlob b

29
backend/src/main/java/com/yoyuzh/files/core/StoredFileEntityRepository.java
View File

@@ -4,8 +4,23 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import java.util.Collection;
import java.util.List;
public interface StoredFileEntityRepository extends JpaRepository<StoredFileEntity, Long> {
interface FileEntityLinkStatsProjection {
Long getFileEntityId();
Long getLinkedStoredFileCount();
Long getLinkedOwnerCount();
String getSampleOwnerUsername();
String getSampleOwnerEmail();
}
@Query("""
select count(distinct relation.storedFile.id)
from StoredFileEntity relation
@@ -41,4 +56,18 @@ public interface StoredFileEntityRepository extends JpaRepository<StoredFileEnti
where relation.fileEntity.id = :fileEntityId
""")
String findSampleOwnerEmailByFileEntityId(@Param("fileEntityId") Long fileEntityId);
@Query("""
select relation.fileEntity.id as fileEntityId,
count(distinct relation.storedFile.id) as linkedStoredFileCount,
count(distinct owner.id) as linkedOwnerCount,
min(owner.username) as sampleOwnerUsername,
min(owner.email) as sampleOwnerEmail
from StoredFileEntity relation
join relation.storedFile storedFile
join storedFile.user owner
where relation.fileEntity.id in :fileEntityIds
group by relation.fileEntity.id
""")
List<FileEntityLinkStatsProjection> findAdminLinkStatsByFileEntityIds(@Param("fileEntityIds") Collection<Long> fileEntityIds);
}

15
backend/src/main/java/com/yoyuzh/files/core/StoredFileRepository.java
View File

@@ -8,11 +8,18 @@ import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import java.time.LocalDateTime;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
public interface StoredFileRepository extends JpaRepository<StoredFile, Long> {
interface UserStorageUsageProjection {
Long getUserId();
Long getUsedStorageBytes();
}
@EntityGraph(attributePaths = {"user", "blob"})
Page<StoredFile> findAllByOrderByCreatedAtDesc(Pageable pageable);
@@ -104,6 +111,14 @@ public interface StoredFileRepository extends JpaRepository<StoredFile, Long> {
""")
long sumFileSizeByUserId(@Param("userId") Long userId);
@Query("""
select f.user.id as userId, coalesce(sum(f.size), 0) as usedStorageBytes
from StoredFile f
where f.user.id in :userIds and f.directory = false and f.deletedAt is null
group by f.user.id
""")
List<UserStorageUsageProjection> sumFileSizeByUserIds(@Param("userIds") Collection<Long> userIds);
@Query("""
select coalesce(sum(f.size), 0)
from StoredFile f

63
backend/src/test/java/com/yoyuzh/admin/AdminInspectionQueryServiceTest.java
View File

@@ -5,7 +5,10 @@ import com.yoyuzh.auth.User;
import com.yoyuzh.auth.UserRepository;
import com.yoyuzh.auth.UserRole;
import com.yoyuzh.common.PageResponse;
import com.yoyuzh.files.core.FileBlob;
import com.yoyuzh.files.core.FileBlobRepository;
import com.yoyuzh.files.core.FileEntity;
import com.yoyuzh.files.core.FileEntityType;
import com.yoyuzh.files.core.FileEntityRepository;
import com.yoyuzh.files.core.StoredFile;
import com.yoyuzh.files.core.StoredFileEntityRepository;
@@ -25,6 +28,9 @@ import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
@@ -123,6 +129,63 @@ class AdminInspectionQueryServiceTest {
assertThat(response.items().get(0).ownerUsername()).isEqualTo("alice");
}
@Test
void shouldListFileBlobsWithBatchLoadedBlobAndLinkStats() {
User creator = createUser(9L, "creator", "creator@example.com");
FileEntity entity = new FileEntity();
entity.setId(100L);
entity.setObjectKey("blobs/a");
entity.setEntityType(FileEntityType.VERSION);
entity.setStoragePolicyId(5L);
entity.setSize(1024L);
entity.setContentType("application/pdf");
entity.setReferenceCount(1);
entity.setCreatedBy(creator);
entity.setCreatedAt(LocalDateTime.now().minusMinutes(2));
FileBlob blob = new FileBlob();
blob.setId(88L);
blob.setObjectKey("blobs/a");
blob.setContentType("application/pdf");
blob.setSize(1024L);
blob.setCreatedAt(LocalDateTime.now().minusMinutes(3));
StoredFileEntityRepository.FileEntityLinkStatsProjection linkStats = mock(StoredFileEntityRepository.FileEntityLinkStatsProjection.class);
when(linkStats.getFileEntityId()).thenReturn(100L);
when(linkStats.getLinkedStoredFileCount()).thenReturn(1L);
when(linkStats.getLinkedOwnerCount()).thenReturn(1L);
when(linkStats.getSampleOwnerUsername()).thenReturn("alice");
when(linkStats.getSampleOwnerEmail()).thenReturn("alice@example.com");
when(fileEntityRepository.searchAdminEntities(anyString(), any(), anyString(), any(), any()))
.thenReturn(new PageImpl<>(List.of(entity)));
when(fileBlobRepository.findAllByObjectKeyIn(any())).thenReturn(List.of(blob));
when(storedFileEntityRepository.findAdminLinkStatsByFileEntityIds(any())).thenReturn(List.of(linkStats));
PageResponse<AdminFileBlobResponse> response = adminInspectionQueryService.listFileBlobs(
0,
10,
null,
null,
null,
null
);
assertThat(response.items()).hasSize(1);
assertThat(response.items().get(0).entityId()).isEqualTo(100L);
assertThat(response.items().get(0).blobId()).isEqualTo(88L);
assertThat(response.items().get(0).linkedStoredFileCount()).isEqualTo(1L);
assertThat(response.items().get(0).linkedOwnerCount()).isEqualTo(1L);
assertThat(response.items().get(0).sampleOwnerUsername()).isEqualTo("alice");
assertThat(response.items().get(0).sampleOwnerEmail()).isEqualTo("alice@example.com");
verify(fileBlobRepository, never()).findByObjectKey(anyString());
verify(storedFileEntityRepository, never()).countByFileEntityId(any());
verify(storedFileEntityRepository, never()).countDistinctOwnersByFileEntityId(any());
verify(storedFileEntityRepository, never()).findSampleOwnerUsernameByFileEntityId(any());
verify(storedFileEntityRepository, never()).findSampleOwnerEmailByFileEntityId(any());
}
private User createUser(Long id, String username, String email) {
User user = new User();
user.setId(id);

6
backend/src/test/java/com/yoyuzh/admin/AdminUserGovernanceServiceTest.java
View File

@@ -27,6 +27,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -67,9 +68,12 @@ class AdminUserGovernanceServiceTest {
@Test
void shouldListUsersWithPagination() {
User user = createUser(1L, "alice", "alice@example.com");
StoredFileRepository.UserStorageUsageProjection usageProjection = mock(StoredFileRepository.UserStorageUsageProjection.class);
when(usageProjection.getUserId()).thenReturn(1L);
when(usageProjection.getUsedStorageBytes()).thenReturn(2048L);
when(userRepository.searchByUsernameOrEmail(anyString(), any()))
.thenReturn(new PageImpl<>(List.of(user)));
when(storedFileRepository.sumFileSizeByUserId(1L)).thenReturn(2048L);
when(storedFileRepository.sumFileSizeByUserIds(any())).thenReturn(List.of(usageProjection));
PageResponse<AdminUserResponse> response = adminUserGovernanceService.listUsers(0, 10, "alice");