Need to upload file on S3 in Java

Question

I have started working on AWS recently. I am currently working on developing upload functionality to S3 storage.

As per my understanding there could be 2 ways to upload a file to S3:-

1. Client's file gets uploaded to my server and i upload this file to S3 server using my credentials. [i will also able to hide this from client as i will not be showing the upload details.]

2. Upload directly to S3

I was able to implement the first approach using simple upload api , but i want to skip the "write uploaded file to server disk" part and stream the content directly to S3 storage, while saving the upload details in my database. I also want to achieve the abstraction of AWS details. How do i do it ?? Please help. Thanks in advance

Answer 1

Try to use POST Object. You do not need to write any code to use it. See more detail in http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-UsingHTTPPOST.html

If you are not afraid of javascript, another opetion is Javascript SDK for S3. You can run it in browsers. http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/frames.html

Presigned url also works, you can generate the url either in browser (by javascript) or backend, and let the browser to send the request to S3 directly.

Answer 2

Create a cfg file on Linux server

vim ~/.aws/credentials

[default]
aws_access_key_id=AKIAIRVAQ7MASDFRTY
aws_secret_access_key=PSIAiFv1Fj/2DySyUQWeCfTyosEoOLwqdmqrRlI

And Save that file. Write the code in NetBeans. Also add libraries:

httpcore-4.4.9.jar
aws-java-sdk-s3-1.11.367.jar
aws-java-sdk-kms-1.11.367.jar
aws-java-sdk-core-1.11.367.jar
aws-java-sdk-1.4.0.1.jar
joda-time-2.8.1.jar
jmespath-java-1.11.367.jar
jackson-dataformat-cbor-2.6.7.jar
jackson-databind-2.6.7.1.jar
jackson-core-2.6.7.jar
jackson-annotations-2.6.0.jar
ion-java-1.0.2.jar
httpcore-4.3.2.jar
httpclient-4.5.5.jar
commons-logging-1.1.3.jar
commons-codec-1.10.jar

package tests3;

 /**
*
 * @author ravi.tyagi
*/
import java.io.File;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.transfer.Download;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;

public class TestS3 {




    public static void main(String[] args) throws Exception {
        String bucketName = "*******BucketName*******";
        String keyName = "recordings/0035db40bb32f4d95f0f23b4514d123g/videos/output/0035db40bb32f4d95f0f23b4514d123g"; // Also the name path from which you want save on S3 without Extension it will be same source file
        String filePath = "/usr/local/WowzaStreamingEngine-4.7.6/content/0035db40bb32f4d95f0f23b4514d123g.mp4";
        try {
            AmazonS3 s3Client = new AmazonS3Client(DefaultAWSCredentialsProviderChain.getInstance());
            TransferManager tm = TransferManagerBuilder.standard().withS3Client(s3Client).build();
            Upload upload = tm.upload(bucketName, keyName, new File(filePath));  
            System.out.println("---****************--->"+s3Client.getUrl(bucketName, "0035db40bb32f4d95f0f23b4514d123g.mp4").toString());
            upload.waitForCompletion();
            System.out.println("---*--->"+s3Client.getUrl(bucketName, "0035db40bb32f4d95f0f23b4514d123g.mp4").toString());
            System.out.println("--**->"+upload.getProgress());
            System.out.println("---***--->"+s3Client.getUrl(bucketName, "0035db40bb32f4d95f0f23b4514d123g.mp4").toString());
            System.out.println("--****->"+upload.getState());
            System.out.println("---*****--->"+s3Client.getUrl(bucketName, "0035db40bb32f4d95f0f23b4514d123g.mp4").toString());
            System.out.println("--******->0");
            System.out.println("Object upload complete");
            System.out.println("--**********-->1");
            tm.shutdownNow();
        } catch (AmazonServiceException e) {
            System.out.println("Exception :- "+e);
        }
        System.out.println("----->2");

    }

}

Answer 3

I am using using byte stream array to write file data to S3 object.

Code for servlet which is receiving uploaded file:-

import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;

import com.src.code.s3.S3FileUploader;

public class FileUploadHandler extends HttpServlet {

    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        doPost(request, response);
    }

    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        PrintWriter out = response.getWriter();

        try{
            List<FileItem> multipartfiledata = new ServletFileUpload(new DiskFileItemFactory()).parseRequest(request);

            //upload to S3
            S3FileUploader s3 = new S3FileUploader();
            String result = s3.fileUploader(multipartfiledata);

            out.print(result);
        } catch(Exception e){
            System.out.println(e.getMessage());
        }
    }
}

Code which is uploading this data as AWS object:-

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
import java.util.UUID;

import org.apache.commons.fileupload.FileItem;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.ClasspathPropertiesFileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;

public class S3FileUploader {


    private static String bucketName     = "***NAME OF YOUR BUCKET***";
    private static String keyName        = "Object-"+UUID.randomUUID();

    public String fileUploader(List<FileItem> fileData) throws IOException {
        AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
        String result = "Upload unsuccessfull because ";
        try {

            S3Object s3Object = new S3Object();

            ObjectMetadata omd = new ObjectMetadata();
            omd.setContentType(fileData.get(0).getContentType());
            omd.setContentLength(fileData.get(0).getSize());
            omd.setHeader("filename", fileData.get(0).getName());

            ByteArrayInputStream bis = new ByteArrayInputStream(fileData.get(0).get());

            s3Object.setObjectContent(bis);
            s3.putObject(new PutObjectRequest(bucketName, keyName, bis, omd));
            s3Object.close();

            result = "Uploaded Successfully.";
        } catch (AmazonServiceException ase) {
           System.out.println("Caught an AmazonServiceException, which means your request made it to Amazon S3, but was "
                + "rejected with an error response for some reason.");

           System.out.println("Error Message:    " + ase.getMessage());
           System.out.println("HTTP Status Code: " + ase.getStatusCode());
           System.out.println("AWS Error Code:   " + ase.getErrorCode());
           System.out.println("Error Type:       " + ase.getErrorType());
           System.out.println("Request ID:       " + ase.getRequestId());

           result = result + ase.getMessage();
        } catch (AmazonClientException ace) {
           System.out.println("Caught an AmazonClientException, which means the client encountered an internal error while "
                + "trying to communicate with S3, such as not being able to access the network.");

           result = result + ace.getMessage();
         }catch (Exception e) {
             result = result + e.getMessage();
       }

        return result;
    }
}

Note :- I am using aws properties file for credentials.

Hope this helps.