A Standalone Java HTTP File Upload Server for handling large text and binary files

Tuesday, March 10, 2009

After I published a code in java for handling file uploads using HTTP POST method, I got several requests for extending the same code to handle binary files like images, executables, etc, in fact I saw some value in extending that code to handle binary files as well as its convenient for me to upload some of my files from one machine to other (in my work) for which I needed an easy lightweight solution, hence this implementation.

In fact its not as easy as I expected, though the implementation is quite simple, but the scalability of this file upload server is not, when I tested it initially with files of small size (less than 10 MB), it used to work perfectly, but for large files, I saw frequent connection resets from browsers especially when the file size is more than 100MB (where I needed to refresh the page again for the upload to work) which straightaway motivated me to improve the scalability of this server than accepting it as a limitation and it was a great learning experience indeed on researching the factors which affects the scalability of this HTTP POST Server implementation with sockets.

Though this file upload server in java can upload files of unlimited size, but you need to tune some parameters accordingly, I have found the buffer size settings of TCP Sockets as one and the next is having the Server Thread to sleep for small intervals of time (in milli seconds) before every receive call so that the server can process large input streams from the client without a processing speed mismatch resulting in connection resets, we will see more about this in the explanation of the code below.

Listing 1: HTTPFileUploadServer.java

 1: /*
2: * HTTPFileUploadServer.java
3: * Author: S.Prasanna
4: * @version 1.00
5: */
6:
7: // A File upload server which will handle files of any type and size
8: // (Text as well as binary files including images)
9:
10: import java.io.*;
11: import java.net.*;
12: import java.util.*;
13:
14: public class HTTPFileUploadServer extends Thread {
15:
16: static final String HTML_START =
17: "<html>" +
18: "<title>HTTP POST Server in java</title>" +
19: "<body>";
20:
21: static final String HTML_END =
22: "</body>" +
23: "</html>";
24:
25: Socket connectedClient = null;
26: DataInputStream inFromClient = null;
27: DataOutputStream outToClient = null;
28:
29: public HTTPFileUploadServer(Socket client) {
30: connectedClient = client;
31: }
32:
33: void closeStreams() throws Exception {
34: inFromClient.close();
35: outToClient.close();
36: connectedClient.close();
37: }
38:
39: // A routine to find the POST request end string from the
40: // Inputstream
41: int sub_array(byte [] array1, byte [] array2) throws Exception {
42:
43: int i = array1.length - 1;
44: int j = array2.length - 1;
45: boolean found = false;
46:
47: for (int k = i; k >=0; k--) {
48: if (array1[k] == array2[j]) {
49: found = true;
50: for (int l = j - 1; l >=0; l--) {
51: k = k - 1;
52: if (k < 0) return -1;
53: if (array1[k] == array2[l]) continue;
54: else {found = false; break;}
55: }
56: if (found == true) return k;
57: }
58: }
59: return -1;
60: }
61:
62: // Read from InputStream
63: public String readLine() throws Exception {
64: String line = "";
65:
66: char c = (char) inFromClient.read();
67:
68: while (c != '\n'){
69: line = line + Character.toString(c);
70: c = (char) (inFromClient.read());
71: }
72: return line.substring(0,line.lastIndexOf('\r'));
73: }
74:
75: //Thread for processing individual clients
76: public void run() {
77:
78: String currentLine = null, postBoundary = null,
79: contentength = null, filename = null, contentLength = null;
80: FileOutputStream fout = null;
81:
82: // Change these two parameters depending on the size of the file to be uploaded
83: // For a very large file > 200 MB, increase THREAD_SLEEP_TIME to prevent connection
84: // resets during upload, current settings are good tor handling upto 100 MB file size
85: long THREAD_SLEEP_TIME = 20;
86: int BUFFER_SIZE = 65535;
87:
88: // Upload File size limit = 25MB, can be increased
89: long FILE_SIZE_LIMIT = 25000000;
90:
91: try {
92:
93: System.out.println( "The Client "+
94: connectedClient.getInetAddress() + ":" + connectedClient.getPort() + " is connected");
95:
96: inFromClient = new DataInputStream(connectedClient.getInputStream());
97: outToClient = new DataOutputStream(connectedClient.getOutputStream());
98:
99: connectedClient.setReceiveBufferSize(BUFFER_SIZE);
100:
101: currentLine = readLine();
102: String headerLine = currentLine;
103: StringTokenizer tokenizer = new StringTokenizer(headerLine);
104: String httpMethod = tokenizer.nextToken();
105: String httpQueryString = tokenizer.nextToken();
106:
107: System.out.println(currentLine);
108:
109: if (httpMethod.equals("GET")) { // GET Request
110: System.out.println("GET request");
111: if (httpQueryString.equals("/")) {
112: // The default home page
113: String responseString = HTTPFileUploadServer.HTML_START +
114: "<form action=\"http://127.0.0.1:5000\" enctype=\"multipart/form-data\"" +
115: "method=\"post\">" +
116: "Enter the name of the File <input name=\"file\" type=\"file\"><br>" +
117: "<input value=\"Upload\" type=\"submit\"></form>" +
118: "Upload only text files." +
119: HTTPFileUploadServer.HTML_END;
120: sendResponse(200, responseString , false);
121: } else {
122: sendResponse(404, "<b>The Requested resource not found ...." +
123: "Usage: http://127.0.0.1:5000", false);
124: }
125:
126: } //if
127: else { //POST Request
128:
129: System.out.println("POST request");
130: while(true) {
131: currentLine = readLine();
132:
133: if (currentLine.indexOf("Content-Type: multipart/form-data") != -1) {
134: postBoundary = currentLine.split("boundary=")[1];
135: // The POST boundary
136:
137: while (true) {
138: currentLine = readLine();
139: if (currentLine.indexOf("Content-Length:") != -1) {
140: contentLength = currentLine.split(" ")[1];
141: System.out.println("Content Length = " + contentLength);
142: break;
143: }
144: }
145:
146: //Content length should be <= 25MB
147: if (Long.valueOf(contentLength) > FILE_SIZE_LIMIT) {
148: inFromClient.skip(Long.valueOf(contentLength));
149: sendResponse(200, "File size should be less than 25MB", false);
150: break;
151: }
152:
153: while (true) {
154: currentLine = readLine();
155: System.out.println(currentLine);
156: if (currentLine.indexOf("--" + postBoundary) != -1) {
157: filename = readLine().split("filename=")[1].replaceAll("\"", "");
158: String [] filelist = filename.split("\\" + System.getProperty("file.separator"));
159: filename = filelist[filelist.length - 1];
160: filename = filename.trim();
161: break;
162: }
163: }
164:
165: if (filename.length() == 0) {
166: System.out.println("No input file selected.");
167: sendResponse(200, "Please select a valid file to upload..", false);
168: break;
169: }
170:
171: String fileContentType = null;
172:
173: try {
174: fileContentType = readLine().split(" ")[1];
175: } catch (Exception e) {
176: System.out.println("Can't determine POST request length");
177: }
178:
179: System.out.println("File content type = " + fileContentType);
180:
181: readLine(); //assert(readLine(inFromClient).equals("")) : "Expected line in POST request is "" ";
182: fout = new FileOutputStream(filename);
183:
184: byte [] buffer = new byte[BUFFER_SIZE], endarray;
185: String end_flag = "--" + postBoundary + "--";
186:
187: endarray = end_flag.getBytes();
188:
189: int bytesRead, bytesAvailable;
190:
191: while ((bytesAvailable = inFromClient.available()) > 0) {
192:
193: Thread.sleep(THREAD_SLEEP_TIME);
194:
195: //System.out.println("Available = " + inFromClient.available());
196: bytesRead = inFromClient.read(buffer, 0, BUFFER_SIZE);
197:
198: int end_byte = 0;
199:
200: //When number of bytes to be read in the stream <>
201: if (bytesAvailable < BUFFER_SIZE) {
202:
203: //System.out.println("End array length =" + endarray.length);
204: //System.out.println("Bytes read = " + bytesRead);
205:
206: // Case where part of POST Boundary comes in the last buffer
207: if (bytesAvailable < endarray.length) {
208: byte [] extendedArray = new byte[BUFFER_SIZE + bytesAvailable];
209: System.arraycopy(buffer, 0, extendedArray, 0, bytesRead);
210: bytesRead = inFromClient.read(extendedArray, BUFFER_SIZE, bytesAvailable);
211: end_byte = sub_array(extendedArray, endarray);
212: if (end_byte == -1) fout.write(buffer, 0, bytesRead);
213: else fout.write(extendedArray, 0, end_byte - 2);
214: }
215: else {
216: // Case where POST Boundary is part of last buffer
217: end_byte = sub_array(buffer, endarray);
218: System.out.println("End byte = " + end_byte);
219: if (end_byte == -1) fout.write(buffer, 0, bytesRead);
220: else fout.write(buffer,0, end_byte - 2);
221: }
222: } else {
223: // Case where POST Boundary is part of the full buffer
224: if (bytesAvailable == 65535) end_byte = sub_array(buffer, endarray);
225: else end_byte = sub_array(buffer, endarray);
226: if (end_byte == -1) fout.write(buffer, 0, bytesRead);
227: else fout.write(buffer,0, end_byte - 2);
228: }
229: }//while
230:
231: sendResponse(200, "File " + filename + " Uploaded..", false);
232: fout.close();
233: break;
234: } //if
235: }//while (true); //End of do-while
236: }//else
237: //Close all streams
238: System.out.println("Closing All Streams....");
239: closeStreams();
240: } catch (Exception e) {
241: e.printStackTrace();
242: }
243: System.out.println("Done....");
244: }
245:
246: public void sendResponse(int statusCode, String responseString, boolean isFile) throws Exception {
247:
248: String statusLine = null;
249: String serverdetails = "Server: Java HTTPServer";
250: String contentLengthLine = null;
251: String fileName = null;
252: String contentTypeLine = "Content-Type: text/html" + "\r\n";
253: FileInputStream fin = null;
254:
255: if (statusCode == 200)
256: statusLine = "HTTP/1.1 200 OK" + "\r\n";
257: else
258: statusLine = "HTTP/1.1 404 Not Found" + "\r\n";
259:
260: if (isFile) {
261: fileName = responseString;
262: fin = new FileInputStream(fileName);
263: contentLengthLine = "Content-Length: " + Integer.toString(fin.available()) + "\r\n";
264: if (!fileName.endsWith(".htm") && !fileName.endsWith(".html"))
265: contentTypeLine = "Content-Type: \r\n";
266: }
267: else {
268: responseString = HTTPFileUploadServer.HTML_START + responseString + HTTPFileUploadServer.HTML_END;
269: contentLengthLine = "Content-Length: " + responseString.length() + "\r\n";
270: }
271:
272: outToClient.writeBytes(statusLine);
273: outToClient.writeBytes(serverdetails);
274: outToClient.writeBytes(contentTypeLine);
275: outToClient.writeBytes(contentLengthLine);
276: outToClient.writeBytes("Connection: close\r\n");
277: outToClient.writeBytes("\r\n");
278:
279: if (isFile) sendFile(fin);
280: else outToClient.writeBytes(responseString);
281: }
282:
283: //Send the requested file
284: public void sendFile(FileInputStream fin) throws Exception {
285: byte[] buffer = new byte[1024] ;
286: int bytesRead;
287:
288: while ((bytesRead = fin.read(buffer)) != -1 ) {
289: outToClient.write(buffer, 0, bytesRead);
290: }
291: fin.close();
292: }
293:
294: public static void main (String args[]) throws Exception {
295:
296: ServerSocket Server = new ServerSocket (5000, 10, InetAddress.getByName("127.0.0.1"));
297: System.out.println ("HTTP Server Waiting for client on port 5000");
298:
299: //Create a new thread for processing clients
300: while(true) {
301: Socket connected = Server.accept();
302: (new HTTPFileUploadServer(connected)).start();
303: }
304: }
305: }
Explanation:

Well, this is a large chunk of code and most part of it are similar to the HTTP POST Server for uploading text files we have seen in an earlier post, the additional things required for handling binary files are

1. Two parameters THREAD_SLEEP_TIME (line 85)and BUFFER_SIZE (line 86) which need to be tuned for handling file uploads of different size, for large file uploads increase the THREAD_SLEEP_TIME so that the connection between the server and the client won't be reset due to stream processing speed mismatch between client and the server, also the current settings are good for handling files of upto 100MB, even though I have restricted the default file upload size limit to 25MB (line 89).

2. Code for tracking the end of the binary stream, this can be done easily using the BufferedReader's readLine method for text files, but for binary files, this method doesn't convert bytes to String properly, therefore we need to process the stream in a byte array (line 184) and check if the POST boundary string (used to indicate the end of the stream in a HTTP POST request) is reached by converting the POST boundary end string into a byte array (line 187) and comparing if the end array is a subset of the byte array read from the inputStream (function sub_array in line 41).

3. At times (very rarely) you may get Connection Reset error from your browser while uploading very large files using this code, but when you get those errors, refresh the page or try uploading again, it should be successful, also modify the parameters mentioned in (1) according to your system's processing power and memory.

4. Lines 200 - 229, here there are three cases which need to be checked for detecting the end of the stream, first we need to check if the bytes available for reading from the stream is < BUFFER_SIZE (line 201), where we can know for sure that this is the last buffer to be read and if the bytes available is less than the end stream array, that means that we read only a part of the end stream in the current buffer, therefore we need to expand the array and get the remaining bytes to track the end stream in the buffer, the second and third cases are straightforward where we just check the end stream in the read buffer to finish processing.

5. The DataInputStream's inherited available method is used to check the end of the input stream from the client (line 191).

Last but not the least, I have tested the code rigorously with simultaneous file uploads with size greater than 100 MB and it worked for me with the above settings, also I have tested this code in IE and Firefox, please feel free to report bugs in this, if any.

11 comments:

Ivan said...

Nice, and works great!
FYI, there is no need to wrap the streams into DataInputStream and DataOutputStream.
Instead, you can write a String to an OutputStream like this:
outToClient.write(statusLine.getBytes(Charset.forName("UTF-8")));

Prasanna Seshadri said...

Hi,

Thanks for saying so, your code is really new to me, will try that out if I can avoid wrapping streams.

Durai said...

I'm trying to run this program on linux server and trying to access from windows machine using IE and Firefox, I just get page can't be displayed. where as it works fine on Linux or MAc desktop with Firefox.

any idea why it doesn work well on windows machine.

Prasanna Seshadri said...

I am not sure about why it doesn't work in that case, in fact I tried it in Windows.

Anonymous said...

Can you please tell me how to get to work with me?

I have eclipse and apache. When I executed your code it displayed server working, then I created a form with method get and enctype multipart/form-data and a submit button, the result is that it gives me get request then close streams then done please help

Anonymous said...

Sorry I got it to work now Thank you very much.

Anonymous said...

Can you please show us how to do it with more than one upload. Please help thank you

Anonymous said...

I tried to use the HTTPCLient from apache to upload the file.
But i got problem by setting the boundary.
methodPost.setRequestHeader("Content-Type", "multipart/form-data; boundary=123456789");

The Httpclient doesn`t use the boundary,which i gave. A temp boundary from httpclient is buided and used.

jxpod said...

How is this server handle if multiple files are uplodaed or multiple clients trying to upload same time.?

Anonymous said...

chrome header is not ordered like this server predict so here is a fix for the code in order to have it work for chrome, firefox and ie.

else { //POST Request

System.out.println("POST request");
boolean fileupload=false;

while(true) {

currentLine = readLine();

if (currentLine.indexOf("Content-Length:") != -1) {
contentLength = currentLine.split(" ")[1];
System.out.println("Content Length = " + contentLength);
}

if (currentLine.indexOf("Content-Type: multipart/form-data") != -1) {
postBoundary = currentLine.split("boundary=")[1];
fileupload=true;
} //if

if(currentLine.equals("")){break;}

}//while (true); //End of do-while
if(fileupload){

// The POST boundary

//Content length should be <= 25MB
if (Long.valueOf(contentLength) < FILE_SIZE_LIMIT) {

while (true) {
currentLine = readLine();
System.out.println(currentLine);
if (currentLine.indexOf("--" + postBoundary) != -1) {
filename = readLine().split("filename=")[1].replaceAll("\"", "");
String [] filelist = filename.split("\\" + System.getProperty("file.separator"));
filename = filelist[filelist.length - 1];
filename = filename.trim();
break;
}
}

if (filename.length() != 0) {

String fileContentType = null;

try {
fileContentType = readLine().split(" ")[1];
} catch (Exception e) {
System.out.println("Can't determine POST request length");
}

System.out.println("File content type = " + fileContentType);

readLine(); //assert(readLine(inFromClient).equals("")) : "Expected line in POST request is "" ";
fout = new FileOutputStream(filename);

byte [] buffer = new byte[BUFFER_SIZE], endarray;
String end_flag = "--" + postBoundary + "--";

endarray = end_flag.getBytes();

int bytesRead, bytesAvailable;

Jatin said...

Can you please elaborate how to use this class to upload any file to server?
I want to upload multiple files and want to download file from server on click of a file link. Thanks


Copyright © 2016 Prasanna Seshadri, www.prasannatech.net, All Rights Reserved.
No part of the content or this site may be reproduced without prior written permission of the author.