fix(storage): integrate s3 session provider
This commit is contained in:
@@ -1,57 +1,61 @@
|
|||||||
package com.yoyuzh.files.storage;
|
package com.yoyuzh.files.storage;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.yoyuzh.common.BusinessException;
|
import com.yoyuzh.common.BusinessException;
|
||||||
import com.yoyuzh.common.ErrorCode;
|
import com.yoyuzh.common.ErrorCode;
|
||||||
import com.yoyuzh.config.FileStorageProperties;
|
import com.yoyuzh.config.FileStorageProperties;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
|
|
||||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
|
||||||
import software.amazon.awssdk.core.ResponseBytes;
|
import software.amazon.awssdk.core.ResponseBytes;
|
||||||
import software.amazon.awssdk.core.sync.RequestBody;
|
import software.amazon.awssdk.core.sync.RequestBody;
|
||||||
import software.amazon.awssdk.http.SdkHttpMethod;
|
import software.amazon.awssdk.http.SdkHttpMethod;
|
||||||
import software.amazon.awssdk.regions.Region;
|
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.S3Client;
|
|
||||||
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
|
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
|
||||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.model.S3Exception;
|
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||||
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
|
|
||||||
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
|
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
|
||||||
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
|
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
|
import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
|
||||||
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
|
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
|
||||||
|
|
||||||
import javax.crypto.Mac;
|
|
||||||
import javax.crypto.spec.SecretKeySpec;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
|
||||||
import java.net.URLEncoder;
|
import java.net.URLEncoder;
|
||||||
import java.net.http.HttpClient;
|
|
||||||
import java.net.http.HttpRequest;
|
|
||||||
import java.net.http.HttpResponse;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.time.Instant;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Iterator;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
public class S3FileContentStorage implements FileContentStorage {
|
public class S3FileContentStorage implements FileContentStorage {
|
||||||
|
|
||||||
private static final String DOGECLOUD_TMP_TOKEN_PATH = "/auth/tmp_token.json";
|
|
||||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||||
|
|
||||||
private final FileStorageProperties.S3 properties;
|
private final FileStorageProperties.S3 properties;
|
||||||
private final HttpClient httpClient = HttpClient.newHttpClient();
|
private final S3SessionProvider sessionProvider;
|
||||||
private TemporaryS3Session cachedSession;
|
|
||||||
|
|
||||||
public S3FileContentStorage(FileStorageProperties storageProperties) {
|
public S3FileContentStorage(FileStorageProperties storageProperties) {
|
||||||
|
this(
|
||||||
|
storageProperties,
|
||||||
|
new DogeCloudS3SessionProvider(
|
||||||
|
storageProperties.getS3(),
|
||||||
|
new DogeCloudTmpTokenClient(storageProperties.getS3(), OBJECT_MAPPER)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
S3FileContentStorage(FileStorageProperties storageProperties,
|
||||||
|
String bucket,
|
||||||
|
software.amazon.awssdk.services.s3.S3Client s3Client,
|
||||||
|
software.amazon.awssdk.services.s3.presigner.S3Presigner s3Presigner) {
|
||||||
|
this(storageProperties, () -> new S3FileRuntimeSession(bucket, s3Client, s3Presigner));
|
||||||
|
}
|
||||||
|
|
||||||
|
S3FileContentStorage(FileStorageProperties storageProperties, S3SessionProvider sessionProvider) {
|
||||||
this.properties = storageProperties.getS3();
|
this.properties = storageProperties.getS3();
|
||||||
|
this.sessionProvider = sessionProvider;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -71,7 +75,9 @@ public class S3FileContentStorage implements FileContentStorage {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public byte[] readFile(Long userId, String path, String storageName) {
|
public byte[] readFile(Long userId, String path, String storageName) {
|
||||||
return readBlob(resolveLegacyFileObjectKey(userId, path, storageName));
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
|
String objectKey = resolveExistingFileObjectKey(session, userId, path, storageName);
|
||||||
|
return readObject(session, objectKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -81,32 +87,33 @@ public class S3FileContentStorage implements FileContentStorage {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String createDownloadUrl(Long userId, String path, String storageName, String filename) {
|
public String createDownloadUrl(Long userId, String path, String storageName, String filename) {
|
||||||
return createBlobDownloadUrl(resolveLegacyFileObjectKey(userId, path, storageName), filename);
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
|
String objectKey = resolveExistingFileObjectKey(session, userId, path, storageName);
|
||||||
|
return createDownloadUrl(session, objectKey, filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PreparedUpload prepareBlobUpload(String path, String filename, String objectKey, String contentType, long size) {
|
public PreparedUpload prepareBlobUpload(String path, String filename, String objectKey, String contentType, long size) {
|
||||||
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
PutObjectRequest.Builder requestBuilder = PutObjectRequest.builder()
|
PutObjectRequest.Builder requestBuilder = PutObjectRequest.builder()
|
||||||
.bucket(getSession().bucket())
|
.bucket(session.bucket())
|
||||||
.key(normalizeObjectKey(objectKey));
|
.key(normalizeObjectKey(objectKey));
|
||||||
if (StringUtils.hasText(contentType)) {
|
if (StringUtils.hasText(contentType)) {
|
||||||
requestBuilder.contentType(contentType);
|
requestBuilder.contentType(contentType);
|
||||||
}
|
}
|
||||||
|
|
||||||
try (S3Presigner presigner = createPresigner()) {
|
PutObjectPresignRequest presignRequest = PutObjectPresignRequest.builder()
|
||||||
PutObjectPresignRequest presignRequest = PutObjectPresignRequest.builder()
|
.signatureDuration(Duration.ofSeconds(Math.max(1, properties.getTtlSeconds())))
|
||||||
.signatureDuration(Duration.ofSeconds(Math.max(1, properties.getTtlSeconds())))
|
.putObjectRequest(requestBuilder.build())
|
||||||
.putObjectRequest(requestBuilder.build())
|
.build();
|
||||||
.build();
|
PresignedPutObjectRequest presignedRequest = session.s3Presigner().presignPutObject(presignRequest);
|
||||||
PresignedPutObjectRequest presignedRequest = presigner.presignPutObject(presignRequest);
|
return new PreparedUpload(
|
||||||
return new PreparedUpload(
|
true,
|
||||||
true,
|
presignedRequest.url().toString(),
|
||||||
presignedRequest.url().toString(),
|
resolveUploadMethod(presignedRequest),
|
||||||
presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : "POST",
|
resolveUploadHeaders(presignedRequest, contentType),
|
||||||
flattenSignedHeaders(presignedRequest.signedHeaders()),
|
objectKey
|
||||||
objectKey
|
);
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -120,13 +127,11 @@ public class S3FileContentStorage implements FileContentStorage {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void completeBlobUpload(String objectKey, String contentType, long size) {
|
public void completeBlobUpload(String objectKey, String contentType, long size) {
|
||||||
try (S3Client s3Client = createClient()) {
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
s3Client.headObject(HeadObjectRequest.builder()
|
try {
|
||||||
.bucket(getSession().bucket())
|
ensureObjectExists(session, normalizeObjectKey(objectKey));
|
||||||
.key(normalizeObjectKey(objectKey))
|
|
||||||
.build());
|
|
||||||
} catch (NoSuchKeyException ex) {
|
} catch (NoSuchKeyException ex) {
|
||||||
throw new BusinessException(ErrorCode.FILE_NOT_FOUND, "File content does not exist");
|
throw new BusinessException(ErrorCode.FILE_NOT_FOUND, "上传文件不存在");
|
||||||
} catch (S3Exception ex) {
|
} catch (S3Exception ex) {
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "File content verification failed");
|
throw new BusinessException(ErrorCode.UNKNOWN, "File content verification failed");
|
||||||
}
|
}
|
||||||
@@ -139,24 +144,15 @@ public class S3FileContentStorage implements FileContentStorage {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public byte[] readBlob(String objectKey) {
|
public byte[] readBlob(String objectKey) {
|
||||||
try (S3Client s3Client = createClient()) {
|
return readObject(sessionProvider.currentSession(), normalizeObjectKey(objectKey));
|
||||||
ResponseBytes<?> response = s3Client.getObjectAsBytes(GetObjectRequest.builder()
|
|
||||||
.bucket(getSession().bucket())
|
|
||||||
.key(normalizeObjectKey(objectKey))
|
|
||||||
.build());
|
|
||||||
return response.asByteArray();
|
|
||||||
} catch (NoSuchKeyException ex) {
|
|
||||||
throw new BusinessException(ErrorCode.FILE_NOT_FOUND, "File content does not exist");
|
|
||||||
} catch (S3Exception ex) {
|
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "File read failed");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void deleteBlob(String objectKey) {
|
public void deleteBlob(String objectKey) {
|
||||||
try (S3Client s3Client = createClient()) {
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
s3Client.deleteObject(DeleteObjectRequest.builder()
|
try {
|
||||||
.bucket(getSession().bucket())
|
session.s3Client().deleteObject(DeleteObjectRequest.builder()
|
||||||
|
.bucket(session.bucket())
|
||||||
.key(normalizeObjectKey(objectKey))
|
.key(normalizeObjectKey(objectKey))
|
||||||
.build());
|
.build());
|
||||||
} catch (S3Exception ex) {
|
} catch (S3Exception ex) {
|
||||||
@@ -166,23 +162,7 @@ public class S3FileContentStorage implements FileContentStorage {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String createBlobDownloadUrl(String objectKey, String filename) {
|
public String createBlobDownloadUrl(String objectKey, String filename) {
|
||||||
GetObjectRequest.Builder requestBuilder = GetObjectRequest.builder()
|
return createDownloadUrl(sessionProvider.currentSession(), normalizeObjectKey(objectKey), filename);
|
||||||
.bucket(getSession().bucket())
|
|
||||||
.key(normalizeObjectKey(objectKey));
|
|
||||||
if (StringUtils.hasText(filename)) {
|
|
||||||
requestBuilder.responseContentDisposition(
|
|
||||||
"attachment; filename*=UTF-8''" + URLEncoder.encode(filename, StandardCharsets.UTF_8)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
try (S3Presigner presigner = createPresigner()) {
|
|
||||||
GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder()
|
|
||||||
.signatureDuration(Duration.ofSeconds(Math.max(1, properties.getTtlSeconds())))
|
|
||||||
.getObjectRequest(requestBuilder.build())
|
|
||||||
.build();
|
|
||||||
PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest);
|
|
||||||
return presignedRequest.url().toString();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -193,6 +173,37 @@ public class S3FileContentStorage implements FileContentStorage {
|
|||||||
public void ensureDirectory(Long userId, String logicalPath) {
|
public void ensureDirectory(Long userId, String logicalPath) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void renameFile(Long userId, String path, String oldStorageName, String newStorageName) {
|
||||||
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
|
String sourceKey = resolveExistingFileObjectKey(session, userId, path, oldStorageName);
|
||||||
|
String targetKey = resolveLegacyFileObjectKey(userId, path, newStorageName);
|
||||||
|
copyObject(session, sourceKey, targetKey);
|
||||||
|
deleteObject(session, sourceKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void moveFile(Long userId, String oldPath, String storageName, String newPath) {
|
||||||
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
|
String sourceKey = resolveExistingFileObjectKey(session, userId, oldPath, storageName);
|
||||||
|
String targetKey = resolveLegacyFileObjectKey(userId, newPath, storageName);
|
||||||
|
copyObject(session, sourceKey, targetKey);
|
||||||
|
deleteObject(session, sourceKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void copyFile(Long userId, String path, String storageName, String targetPath) {
|
||||||
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
|
String sourceKey = resolveExistingFileObjectKey(session, userId, path, storageName);
|
||||||
|
String targetKey = resolveLegacyFileObjectKey(userId, targetPath, storageName);
|
||||||
|
copyObject(session, sourceKey, targetKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void storeImportedFile(Long userId, String path, String storageName, String contentType, byte[] content) {
|
||||||
|
storeBlob(resolveLegacyFileObjectKey(userId, path, storageName), contentType, content);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void storeTransferFile(String sessionId, String storageName, String contentType, byte[] content) {
|
public void storeTransferFile(String sessionId, String storageName, String contentType, byte[] content) {
|
||||||
putObject(resolveTransferObjectKey(sessionId, storageName), contentType, content);
|
putObject(resolveTransferObjectKey(sessionId, storageName), contentType, content);
|
||||||
@@ -220,149 +231,159 @@ public class S3FileContentStorage implements FileContentStorage {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String resolveLegacyFileObjectKey(Long userId, String path, String storageName) {
|
public String resolveLegacyFileObjectKey(Long userId, String path, String storageName) {
|
||||||
return "users/" + userId + "/" + normalizeRelativePath(path) + "/" + normalizeName(storageName);
|
return "users/" + userId + "/" + joinObjectKeyParts(normalizeRelativePath(path), normalizeName(storageName));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveExistingFileObjectKey(S3FileRuntimeSession session, Long userId, String path, String storageName) {
|
||||||
|
String currentKey = resolveLegacyFileObjectKey(userId, path, storageName);
|
||||||
|
try {
|
||||||
|
ensureObjectExists(session, currentKey);
|
||||||
|
return currentKey;
|
||||||
|
} catch (NoSuchKeyException ex) {
|
||||||
|
String legacyKey = userId + "/" + joinObjectKeyParts(normalizeRelativePath(path), normalizeName(storageName));
|
||||||
|
ensureObjectExists(session, legacyKey);
|
||||||
|
return legacyKey;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void putObject(String objectKey, String contentType, byte[] content) {
|
private void putObject(String objectKey, String contentType, byte[] content) {
|
||||||
|
S3FileRuntimeSession session = sessionProvider.currentSession();
|
||||||
PutObjectRequest.Builder requestBuilder = PutObjectRequest.builder()
|
PutObjectRequest.Builder requestBuilder = PutObjectRequest.builder()
|
||||||
.bucket(getSession().bucket())
|
.bucket(session.bucket())
|
||||||
.key(normalizeObjectKey(objectKey));
|
.key(normalizeObjectKey(objectKey));
|
||||||
if (StringUtils.hasText(contentType)) {
|
if (StringUtils.hasText(contentType)) {
|
||||||
requestBuilder.contentType(contentType);
|
requestBuilder.contentType(contentType);
|
||||||
}
|
}
|
||||||
|
|
||||||
try (S3Client s3Client = createClient()) {
|
try {
|
||||||
s3Client.putObject(requestBuilder.build(), RequestBody.fromBytes(content));
|
session.s3Client().putObject(requestBuilder.build(), RequestBody.fromBytes(content));
|
||||||
} catch (S3Exception ex) {
|
} catch (S3Exception ex) {
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "File write failed");
|
throw new BusinessException(ErrorCode.UNKNOWN, "File write failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String resolveTransferObjectKey(String sessionId, String storageName) {
|
private byte[] readObject(S3FileRuntimeSession session, String objectKey) {
|
||||||
return "transfers/" + normalizeName(sessionId) + "/" + normalizeName(storageName);
|
|
||||||
}
|
|
||||||
|
|
||||||
private S3Client createClient() {
|
|
||||||
TemporaryS3Session session = getSession();
|
|
||||||
return S3Client.builder()
|
|
||||||
.endpointOverride(session.endpointUri())
|
|
||||||
.region(Region.of(properties.getRegion()))
|
|
||||||
.credentialsProvider(StaticCredentialsProvider.create(session.credentials()))
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
private S3Presigner createPresigner() {
|
|
||||||
TemporaryS3Session session = getSession();
|
|
||||||
return S3Presigner.builder()
|
|
||||||
.endpointOverride(session.endpointUri())
|
|
||||||
.region(Region.of(properties.getRegion()))
|
|
||||||
.credentialsProvider(StaticCredentialsProvider.create(session.credentials()))
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
private synchronized TemporaryS3Session getSession() {
|
|
||||||
if (cachedSession != null && cachedSession.expiresAt().isAfter(Instant.now().plusSeconds(60))) {
|
|
||||||
return cachedSession;
|
|
||||||
}
|
|
||||||
|
|
||||||
cachedSession = requestTemporaryS3Session();
|
|
||||||
return cachedSession;
|
|
||||||
}
|
|
||||||
|
|
||||||
private TemporaryS3Session requestTemporaryS3Session() {
|
|
||||||
requireText(properties.getApiAccessKey(), "Missing DogeCloud API access key");
|
|
||||||
requireText(properties.getApiSecretKey(), "Missing DogeCloud API secret key");
|
|
||||||
requireText(properties.getScope(), "Missing DogeCloud storage scope");
|
|
||||||
|
|
||||||
String body = "{\"channel\":\"OSS_FULL\",\"ttl\":" + Math.max(1, properties.getTtlSeconds())
|
|
||||||
+ ",\"scopes\":[\"" + escapeJson(properties.getScope()) + "\"]}";
|
|
||||||
HttpRequest request = HttpRequest.newBuilder()
|
|
||||||
.uri(URI.create(trimTrailingSlash(properties.getApiBaseUrl()) + DOGECLOUD_TMP_TOKEN_PATH))
|
|
||||||
.header("Content-Type", "application/json")
|
|
||||||
.header("Authorization", createDogeCloudApiAuthorization(body))
|
|
||||||
.POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
|
ResponseBytes<?> response = session.s3Client().getObjectAsBytes(GetObjectRequest.builder()
|
||||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
.bucket(session.bucket())
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "DogeCloud temporary credential request failed");
|
.key(normalizeObjectKey(objectKey))
|
||||||
}
|
.build());
|
||||||
|
return response.asByteArray();
|
||||||
JsonNode payload = OBJECT_MAPPER.readTree(response.body());
|
} catch (NoSuchKeyException ex) {
|
||||||
if (payload.path("code").asInt() != 200) {
|
throw new BusinessException(ErrorCode.FILE_NOT_FOUND, "File content does not exist");
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "DogeCloud temporary credential request failed");
|
} catch (S3Exception ex) {
|
||||||
}
|
throw new BusinessException(ErrorCode.UNKNOWN, "File read failed");
|
||||||
|
|
||||||
JsonNode data = payload.path("data");
|
|
||||||
JsonNode credentials = data.path("Credentials");
|
|
||||||
JsonNode bucket = selectBucket(data.path("Buckets"), extractScopeBucketName(properties.getScope()));
|
|
||||||
Instant expiresAt = data.hasNonNull("ExpiredAt")
|
|
||||||
? Instant.ofEpochSecond(data.path("ExpiredAt").asLong())
|
|
||||||
: Instant.now().plusSeconds(Math.max(1, properties.getTtlSeconds()));
|
|
||||||
|
|
||||||
return new TemporaryS3Session(
|
|
||||||
requireText(credentials.path("accessKeyId").asText(null), "Missing DogeCloud temporary access key"),
|
|
||||||
requireText(credentials.path("secretAccessKey").asText(null), "Missing DogeCloud temporary secret key"),
|
|
||||||
requireText(credentials.path("sessionToken").asText(null), "Missing DogeCloud temporary session token"),
|
|
||||||
requireText(bucket.path("s3Bucket").asText(null), "Missing DogeCloud S3 bucket"),
|
|
||||||
toEndpointUri(requireText(bucket.path("s3Endpoint").asText(null), "Missing DogeCloud S3 endpoint")),
|
|
||||||
expiresAt
|
|
||||||
);
|
|
||||||
} catch (IOException ex) {
|
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "DogeCloud temporary credential response is invalid");
|
|
||||||
} catch (InterruptedException ex) {
|
|
||||||
Thread.currentThread().interrupt();
|
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "DogeCloud temporary credential request interrupted");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private JsonNode selectBucket(JsonNode buckets, String bucketName) {
|
private String createDownloadUrl(S3FileRuntimeSession session, String objectKey, String filename) {
|
||||||
if (!buckets.isArray() || buckets.isEmpty()) {
|
GetObjectRequest.Builder requestBuilder = GetObjectRequest.builder()
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "DogeCloud temporary credential response has no bucket");
|
.bucket(session.bucket())
|
||||||
|
.key(normalizeObjectKey(objectKey));
|
||||||
|
if (StringUtils.hasText(filename)) {
|
||||||
|
requestBuilder.responseContentDisposition(createContentDisposition(filename));
|
||||||
}
|
}
|
||||||
|
|
||||||
Iterator<JsonNode> iterator = buckets.elements();
|
GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder()
|
||||||
JsonNode first = buckets.get(0);
|
.signatureDuration(Duration.ofSeconds(Math.max(1, properties.getTtlSeconds())))
|
||||||
while (iterator.hasNext()) {
|
.getObjectRequest(requestBuilder.build())
|
||||||
JsonNode bucket = iterator.next();
|
.build();
|
||||||
if (bucketName.equals(bucket.path("name").asText())) {
|
PresignedGetObjectRequest presignedRequest = session.s3Presigner().presignGetObject(presignRequest);
|
||||||
return bucket;
|
return presignedRequest.url().toString();
|
||||||
}
|
|
||||||
}
|
|
||||||
return first;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, String> flattenSignedHeaders(Map<String, java.util.List<String>> headers) {
|
private void copyObject(S3FileRuntimeSession session, String sourceKey, String targetKey) {
|
||||||
|
try {
|
||||||
|
session.s3Client().copyObject(CopyObjectRequest.builder()
|
||||||
|
.sourceBucket(session.bucket())
|
||||||
|
.sourceKey(normalizeObjectKey(sourceKey))
|
||||||
|
.destinationBucket(session.bucket())
|
||||||
|
.destinationKey(normalizeObjectKey(targetKey))
|
||||||
|
.build());
|
||||||
|
} catch (S3Exception ex) {
|
||||||
|
throw new BusinessException(ErrorCode.UNKNOWN, "File copy failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteObject(S3FileRuntimeSession session, String objectKey) {
|
||||||
|
try {
|
||||||
|
session.s3Client().deleteObject(DeleteObjectRequest.builder()
|
||||||
|
.bucket(session.bucket())
|
||||||
|
.key(normalizeObjectKey(objectKey))
|
||||||
|
.build());
|
||||||
|
} catch (S3Exception ex) {
|
||||||
|
throw new BusinessException(ErrorCode.UNKNOWN, "File delete failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ensureObjectExists(S3FileRuntimeSession session, String objectKey) {
|
||||||
|
session.s3Client().headObject(HeadObjectRequest.builder()
|
||||||
|
.bucket(session.bucket())
|
||||||
|
.key(normalizeObjectKey(objectKey))
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveUploadMethod(PresignedPutObjectRequest presignedRequest) {
|
||||||
|
if (presignedRequest.httpRequest() == null) {
|
||||||
|
return "PUT";
|
||||||
|
}
|
||||||
|
return presignedRequest.httpRequest().method() == SdkHttpMethod.PUT ? "PUT" : "POST";
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, String> resolveUploadHeaders(PresignedPutObjectRequest presignedRequest, String contentType) {
|
||||||
|
Map<String, String> headers = flattenSignedHeaders(presignedRequest.signedHeaders());
|
||||||
|
if (StringUtils.hasText(contentType)) {
|
||||||
|
headers.put("Content-Type", contentType);
|
||||||
|
}
|
||||||
|
return headers;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, String> flattenSignedHeaders(Map<String, List<String>> signedHeaders) {
|
||||||
Map<String, String> flattened = new HashMap<>();
|
Map<String, String> flattened = new HashMap<>();
|
||||||
headers.forEach((key, values) -> {
|
if (signedHeaders == null) {
|
||||||
if (!values.isEmpty()) {
|
return flattened;
|
||||||
|
}
|
||||||
|
signedHeaders.forEach((key, values) -> {
|
||||||
|
if (values != null && !values.isEmpty()) {
|
||||||
flattened.put(key, String.join(",", values));
|
flattened.put(key, String.join(",", values));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
return flattened;
|
return flattened;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String createDogeCloudApiAuthorization(String body) {
|
private String createContentDisposition(String filename) {
|
||||||
return "TOKEN " + properties.getApiAccessKey() + ":" + hmacSha1Hex(
|
return "attachment; filename=\"" + createAsciiFallbackFilename(filename)
|
||||||
properties.getApiSecretKey(),
|
+ "\"; filename*=UTF-8''" + URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
|
||||||
DOGECLOUD_TMP_TOKEN_PATH + "\n" + body
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private String hmacSha1Hex(String secret, String value) {
|
private String createAsciiFallbackFilename(String filename) {
|
||||||
try {
|
String fallback = "download";
|
||||||
Mac mac = Mac.getInstance("HmacSHA1");
|
int dotIndex = filename.lastIndexOf('.');
|
||||||
mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA1"));
|
if (dotIndex > 0 && dotIndex < filename.length() - 1) {
|
||||||
byte[] digest = mac.doFinal(value.getBytes(StandardCharsets.UTF_8));
|
String extension = filename.substring(dotIndex);
|
||||||
StringBuilder result = new StringBuilder(digest.length * 2);
|
if (isSafeAsciiToken(extension)) {
|
||||||
for (byte item : digest) {
|
fallback += extension;
|
||||||
result.append(String.format("%02x", item));
|
|
||||||
}
|
}
|
||||||
return result.toString();
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, "DogeCloud authorization signing failed");
|
|
||||||
}
|
}
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isSafeAsciiToken(String value) {
|
||||||
|
for (int index = 0; index < value.length(); index++) {
|
||||||
|
char current = value.charAt(index);
|
||||||
|
if (current < 33 || current > 126 || current == '"' || current == '\\' || current == ';') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveTransferObjectKey(String sessionId, String storageName) {
|
||||||
|
return "transfers/" + normalizeName(sessionId) + "/" + normalizeName(storageName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String joinObjectKeyParts(String path, String storageName) {
|
||||||
|
return StringUtils.hasText(path) ? path + "/" + storageName : storageName;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String normalizeObjectKey(String objectKey) {
|
private String normalizeObjectKey(String objectKey) {
|
||||||
@@ -394,44 +415,4 @@ public class S3FileContentStorage implements FileContentStorage {
|
|||||||
}
|
}
|
||||||
return cleaned;
|
return cleaned;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String extractScopeBucketName(String scope) {
|
|
||||||
int separatorIndex = scope.indexOf(':');
|
|
||||||
return separatorIndex >= 0 ? scope.substring(0, separatorIndex) : scope;
|
|
||||||
}
|
|
||||||
|
|
||||||
private URI toEndpointUri(String endpoint) {
|
|
||||||
return URI.create(endpoint.startsWith("http://") || endpoint.startsWith("https://")
|
|
||||||
? endpoint
|
|
||||||
: "https://" + endpoint);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String trimTrailingSlash(String value) {
|
|
||||||
return value.replaceAll("/+$", "");
|
|
||||||
}
|
|
||||||
|
|
||||||
private String escapeJson(String value) {
|
|
||||||
return value.replace("\\", "\\\\").replace("\"", "\\\"");
|
|
||||||
}
|
|
||||||
|
|
||||||
private String requireText(String value, String message) {
|
|
||||||
if (!StringUtils.hasText(value)) {
|
|
||||||
throw new BusinessException(ErrorCode.UNKNOWN, message);
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
private record TemporaryS3Session(
|
|
||||||
String accessKeyId,
|
|
||||||
String secretAccessKey,
|
|
||||||
String sessionToken,
|
|
||||||
String bucket,
|
|
||||||
URI endpointUri,
|
|
||||||
Instant expiresAt
|
|
||||||
) {
|
|
||||||
|
|
||||||
AwsSessionCredentials credentials() {
|
|
||||||
return AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -450,3 +450,4 @@ Android 壳补充说明:
|
|||||||
- 2026-04-08 阶段 3 第三小步补充:上传会话新增 part 状态记录。`UploadSessionService.recordUploadedPart()` 会校验会话归属、状态、过期时间和 part 范围,把 `etag/size/uploadedAt` 写入 `uploadedPartsJson`,并将新会话推进到 `UPLOADING`。当前实现是会话状态跟踪,不是跨存储驱动的分片内容写入/合并实现。
|
- 2026-04-08 阶段 3 第三小步补充:上传会话新增 part 状态记录。`UploadSessionService.recordUploadedPart()` 会校验会话归属、状态、过期时间和 part 范围,把 `etag/size/uploadedAt` 写入 `uploadedPartsJson`,并将新会话推进到 `UPLOADING`。当前实现是会话状态跟踪,不是跨存储驱动的分片内容写入/合并实现。
|
||||||
- 2026-04-08 阶段 3 第四小步补充:上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话,尝试删除 `objectKey` 对应的临时 blob,然后标记为 `EXPIRED`。已完成文件不参与清理,避免误删已经落库的生产对象。
|
- 2026-04-08 阶段 3 第四小步补充:上传会话新增定时过期清理。`UploadSessionService.pruneExpiredSessions()` 每小时扫描未完成且已过期的 `CREATED/UPLOADING/COMPLETING` 会话,尝试删除 `objectKey` 对应的临时 blob,然后标记为 `EXPIRED`。已完成文件不参与清理,避免误删已经落库的生产对象。
|
||||||
- 2026-04-08 阶段 4 第一小步补充:后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略,并把当前 `FileStorageProperties` 映射为 `LOCAL` 或 `S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON;当前能力声明中 `multipartUpload=false`,用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID,但 `FileContentStorage` 仍保持单对象上传/校验抽象,旧 `/api/files/**` 生产路径不切换。
|
- 2026-04-08 阶段 4 第一小步补充:后端新增存储策略骨架。`StoragePolicyService` 作为 `CommandLineRunner` 在启动时确保存在默认策略,并把当前 `FileStorageProperties` 映射为 `LOCAL` 或 `S3_COMPATIBLE` 策略及 `StoragePolicyCapabilities` JSON;当前能力声明中 `multipartUpload=false`,用于明确真实对象存储分片写入/合并还没有启用。`UploadSession.storagePolicyId` 开始记录默认策略 ID,但 `FileContentStorage` 仍保持单对象上传/校验抽象,旧 `/api/files/**` 生产路径不切换。
|
||||||
|
- 2026-04-08 `files/storage` 合并补充:S3 存储实现拆出多吉云临时密钥客户端与运行期会话提供器。`S3FileContentStorage` 现在通过 `S3SessionProvider.currentSession()` 获取当前 bucket、`S3Client` 和 `S3Presigner`,避免每次操作重复内联多吉云 token 解析逻辑;测试环境可直接注入 mock S3 client/presigner。该改动没有引入 multipart,仍是单对象 PUT/HEAD/GET/COPY/DELETE 路径。
|
||||||
|
|||||||
@@ -165,3 +165,4 @@
|
|||||||
- 2026-04-08 阶段 3 第四小步:`UploadSessionService` 新增定时过期清理,按小时扫描 `CREATED/UPLOADING/COMPLETING` 且已过期的会话,尝试删除对应临时 `blobs/...` 对象,并把会话标记为 `EXPIRED`;`COMPLETED/CANCELLED/FAILED/EXPIRED` 不在本轮清理范围内。
|
- 2026-04-08 阶段 3 第四小步:`UploadSessionService` 新增定时过期清理,按小时扫描 `CREATED/UPLOADING/COMPLETING` 且已过期的会话,尝试删除对应临时 `blobs/...` 对象,并把会话标记为 `EXPIRED`;`COMPLETED/CANCELLED/FAILED/EXPIRED` 不在本轮清理范围内。
|
||||||
- 2026-04-08 multipart 评估结论:暂不把 v2 上传会话直接接入真实对象存储分片写入/合并。当前 `FileContentStorage` 仍是单对象上传/校验抽象,缺少 multipart uploadId、part URL 预签名、complete/abort 语义;立即接入会把上传会话写死在当前多吉云 S3 配置上,并让过期清理误以为 `deleteBlob` 能释放未完成分片。下一步先做阶段 4 存储策略与能力声明骨架,再按 `multipartUpload` 能力接 S3 multipart。
|
- 2026-04-08 multipart 评估结论:暂不把 v2 上传会话直接接入真实对象存储分片写入/合并。当前 `FileContentStorage` 仍是单对象上传/校验抽象,缺少 multipart uploadId、part URL 预签名、complete/abort 语义;立即接入会把上传会话写死在当前多吉云 S3 配置上,并让过期清理误以为 `deleteBlob` 能释放未完成分片。下一步先做阶段 4 存储策略与能力声明骨架,再按 `multipartUpload` 能力接 S3 multipart。
|
||||||
- 2026-04-08 阶段 4 第一小步:新增 `StoragePolicy`、`StoragePolicyType`、`StoragePolicyCredentialMode`、`StoragePolicyCapabilities` 与 `StoragePolicyService`,启动时把当前 `app.storage.provider` 映射成一条默认策略;本地策略声明 `serverProxyDownload=true`、`multipartUpload=false`,多吉云/S3 兼容策略声明 `directUpload=true`、`signedDownloadUrl=true`、`requiresCors=true`、`multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`,但旧上传下载路径和前端上传队列仍未切换。
|
- 2026-04-08 阶段 4 第一小步:新增 `StoragePolicy`、`StoragePolicyType`、`StoragePolicyCredentialMode`、`StoragePolicyCapabilities` 与 `StoragePolicyService`,启动时把当前 `app.storage.provider` 映射成一条默认策略;本地策略声明 `serverProxyDownload=true`、`multipartUpload=false`,多吉云/S3 兼容策略声明 `directUpload=true`、`signedDownloadUrl=true`、`requiresCors=true`、`multipartUpload=false`。新 v2 上传会话会记录默认 `storagePolicyId`,但旧上传下载路径和前端上传队列仍未切换。
|
||||||
|
- 2026-04-08 合并 `files/storage` 补提交后修复:`S3FileContentStorage` 改为复用 `DogeCloudS3SessionProvider` / `DogeCloudTmpTokenClient` 获取并缓存运行期 `S3Client` 与 `S3Presigner`,保留生产构造器 `S3FileContentStorage(FileStorageProperties)`,同时提供测试用注入构造器;S3 直传、签名下载、上传校验、读旧对象键 fallback、rename/move/copy、离线快传对象读写继续通过 `FileContentStorage` 统一抽象。
|
||||||
|
|||||||
Reference in New Issue
Block a user