0
私は、JavaプログラムからAWS S3にストリーミングデータをアップロードする必要があります。私のプログラムは以下の通りです。私はストリームをアップロードする際に部品のサイズを知らない。AWS S3 Java SDK
if(!islastChunk)
uploadRequest.withPartSize(5242880);
else
uploadRequest.setLastPart(islastChunk);
lastPartがfalseで、私は最後の最後の部分のためにUploadPartResultに真としてsetLastPartを設定した場合ので、私は5メガバイトとしてsetPartSizeするロジックの上に持っています。
package com.aws.s3;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
public class MultipartFileUpload {
public static void main(String[] args) {
AWSCredentials credentials = new BasicAWSCredentials("ABC", "DEF");
AmazonS3 s3 = new AmazonS3Client(credentials);
String existingBucketName = "streamdatastore";
String keyName = "PeddlerGuideData.txt";
List<PartETag> partETags = new ArrayList<PartETag>();
//String filePath = "";
int[] idx = { 1 };
InitiateMultipartUploadResult initResponse= initChunkDataUpload(s3,existingBucketName,keyName);
try {
String content = new String(Files.readAllBytes(Paths.get("MultiPartTestData.txt")));
InputStream is = new ByteArrayInputStream(content.getBytes());
uploadChunkData(s3,initResponse,initResponse.getBucketName(),initResponse.getKey(),1,is,partETags,false);
uploadChunkData(s3,initResponse,initResponse.getBucketName(),initResponse.getKey(),2,is,partETags,true);
} catch (IOException e) {
e.printStackTrace();
}
completeUploadData(s3,initResponse,initResponse.getBucketName(),initResponse.getKey(),partETags);
}
private static void processLine(String line,AmazonS3 s3,InitiateMultipartUploadResult initResponse,int partNumber,List<PartETag> partETags){
System.out.println("Line number "+ partNumber+"\n line = \n"+line);
InputStream is = new ByteArrayInputStream(line.getBytes());
uploadChunkData(s3,initResponse,initResponse.getBucketName(),initResponse.getKey(),partNumber,is,partETags,false);
}
private static void putMultiPartRequest(){
}
private static InitiateMultipartUploadResult initChunkDataUpload(AmazonS3 s3Client,String existingBucketName,String keyName){
InitiateMultipartUploadRequest initRequest = new
InitiateMultipartUploadRequest(existingBucketName, keyName);
InitiateMultipartUploadResult initResponse =
s3Client.initiateMultipartUpload(initRequest);
return initResponse;
}
private static List<PartETag> uploadChunkData(AmazonS3 s3Client,InitiateMultipartUploadResult initResponse ,String existingBucketName,String keyName,int partNumber,InputStream inputStream,List<PartETag> partETags,boolean islastChunk){
UploadPartRequest uploadRequest = null;
try{
uploadRequest = new UploadPartRequest()
.withBucketName(existingBucketName).withKey(keyName)
.withUploadId(initResponse.getUploadId()).withPartNumber(partNumber).withInputStream(inputStream);
if(!islastChunk)
uploadRequest.withPartSize(5242880);
else
uploadRequest.setLastPart(islastChunk);
UploadPartResult uploadPartResult =s3Client.uploadPart(uploadRequest);
partETags.add(uploadPartResult.getPartETag());
System.out.println("chunk data upload is completed");
}catch(Exception e){
e.printStackTrace();
}finally{
}
return partETags;
}
private static void completeUploadData (AmazonS3 s3Client,InitiateMultipartUploadResult initResponse ,String existingBucketName,String keyName,List<PartETag> partETags){
try{
CompleteMultipartUploadRequest compRequest = new
CompleteMultipartUploadRequest(
existingBucketName,
keyName,
initResponse.getUploadId(),
partETags);
s3Client.completeMultipartUpload(compRequest);
}catch(Exception e){
e.printStackTrace();
}finally{
}
}
}
部品サイズを5 MBに設定すると、次の例外が発生します。
com.amazonaws.services.s3.model.AmazonS3Exception: Your socket connection to the server was not read from or written to within the timeout period. Idle connections will be closed. (Service: Amazon S3; Status Code: 400; Error Code: RequestTimeout; Request ID: 4B6405551FAABAFB), S3 Extended Request ID: v5RWy2tut1mEMzDZ1mwe0bFj6m9ho6eUihuO0p0EsccQtWv2814Ak18jtBNvxqDdt3AHy1otvTI=
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1530)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1168)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:949)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:662)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:636)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:619)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$300(AmazonHttpClient.java:587)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:574)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:446)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4035)
at com.amazonaws.services.s3.AmazonS3Client.doUploadPart(AmazonS3Client.java:3045)
at com.amazonaws.services.s3.AmazonS3Client.uploadPart(AmazonS3Client.java:3030)
at com.aws.s3.MultipartFileUpload.uploadChunkData(MultipartFileUpload.java:81)
at com.aws.s3.MultipartFileUpload.main(MultipartFileUpload.java:44)
上記のロジックを削除すると、以下のエラーが発生します。
com.amazonaws.services.s3.model.AmazonS3Exception: Your proposed upload is smaller than the minimum allowed size (Service: Amazon S3; Status Code: 400; Error Code: EntityTooSmall; Request ID: 11A9870A168EDBE0), S3 Extended Request ID: 7OnccoS26Ijd4SKvBj9mVvlH4SZk4oloNouNS5gc/QyS4DQXm5A24/Ds96WRHdlkr76nAaPSxR4=
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1530)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1168)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:949)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:662)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:636)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:619)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$300(AmazonHttpClient.java:587)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:574)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:446)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4035)
at com.amazonaws.services.s3.AmazonS3Client.completeMultipartUpload(AmazonS3Client.java:2806)
at com.aws.s3.MultipartFileUpload.completeUploadData(MultipartFileUpload.java:103)
at com.aws.s3.MultipartFileUpload.main(MultipartFileUpload.java:49)
どのように同じことを進めるかに関する提案。