Web Server can now handle large files

src/main/java/com/sforce/dataset/DatasetUtilConstants.java

@@ -29,6 +29,7 @@
 
 
 public class DatasetUtilConstants {
+	public static final int DEFAULT_BUFFER_SIZE = 8*1024*1024;
 	public static boolean createNewDateParts = false;
 	public static File currentDir =  new File("").getAbsoluteFile();
 	public static boolean debug = false;

src/main/java/com/sforce/dataset/flow/monitor/DataFlowMonitorUtil.java

@@ -48,6 +48,7 @@
 
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.sforce.dataset.DatasetUtilConstants;
 import com.sforce.soap.partner.PartnerConnection;
 import com.sforce.ws.ConnectionException;
 import com.sforce.ws.ConnectorConfig;
@@ -227,7 +228,7 @@ public static boolean getJobErrorFile(PartnerConnection partnerConnection, Strin
 		InputStream emis = emresponseEntity.getContent();			
 		File outfile = new File(datasetName+"_"+jobTrackerid+"_error.csv");
 		System.out.println("fetching file {"+outfile+"}. Content-length {"+emresponseEntity.getContentLength()+"}");
-		BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outfile));
+		BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outfile),DatasetUtilConstants.DEFAULT_BUFFER_SIZE);
 		IOUtils.copy(emis, out);
 		out.close();
 		emis.close();

src/main/java/com/sforce/dataset/loader/EbinFormatWriter.java

@@ -205,7 +205,10 @@ public void addrow(String[] values)  throws IOException,NumberFormatException, P
 	            	bindings.put("curr", curr);
 	            	bindings.put("prev", prev);
 	            	columnValue = cs.eval(bindings);
-//	            	System.out.println("columnValue:"+columnValue);
+	            	if(DatasetUtilConstants.debug)
+	            	{
+	            		logger.println(_dataTypes.get(key_value_count).getName()+ " Computed columnValue: "+columnValue);
+	            	}
 	            }
 	            }catch(Throwable t)
 	            {

src/main/java/com/sforce/dataset/server/DatasetUtilServer.java

@@ -78,7 +78,7 @@ public void init(String[] args, boolean join, PartnerConnection partnerConnectio
     	DatasetUtilServer.partnerConnection = partnerConnection;
         final String WEBAPPDIR = "index.html";
         final String contextPath = "/";
-        final int maxFormContentSize = 1048576;
+        final int maxFormContentSize = 40 * 1000 * 1024 * 1024;
 
         final Server server = new Server(port);
 

src/main/java/com/sforce/dataset/server/MultipartRequestHandler.java

@@ -25,6 +25,7 @@
  */
 package com.sforce.dataset.server;
 
+import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -120,9 +121,20 @@ public static List<FileUploadRequest> uploadByApacheFileUpload(HttpServletReques
 						ExternalFileSchema.save(outFile, schema, System.out);
 					}else
 					{
-						FileOutputStream out = new FileOutputStream(outFile);
-						IOUtils.copy(fm.inputFileStream, out);
-						out.close();
+						FileOutputStream fos = null;
+						BufferedOutputStream bos = null;
+						try
+						{
+							fos = new FileOutputStream(outFile);
+							bos = new BufferedOutputStream(fos,DatasetUtilConstants.DEFAULT_BUFFER_SIZE);
+							IOUtils.copy(fm.inputFileStream, bos);
+						}finally
+						{
+							if(bos!=null)
+								IOUtils.closeQuietly(bos);
+							if(fos!=null)
+								IOUtils.closeQuietly(fos);
+						}
 					}
 					fm.savedFile = outFile;
 					fm.setInputFileSize(outFile.length()+"");

src/main/java/com/sforce/dataset/util/FiscalDateUtil.java

@@ -209,17 +209,20 @@ public static void main(String[] args) {
 
 	    		minimalDaysInFirstWeek = getminimalDaysInFirstWeek(cal.get(Calendar.DAY_OF_WEEK), firstDayOfWeek);
 
-	    		System.out.println("\nfirstDayOfWeek: "+firstDayOfWeek+" minimalDaysInFirstWeek: "+minimalDaysInFirstWeek);
-
-	    		System.out.println("Date\t\t\t\tFiscal Year\tFiscal Quarter\tCal Quarter\tFiscal Month\tFiscal Week\tCal Week");
+	    		System.out.println("\nfirstDayOfWeek: "+firstDayOfWeek+"\t\tminimalDaysInFirstWeek: "+minimalDaysInFirstWeek+"\t\tfiscalMonthOffset: "+fiscalMonthOffset);
+	    		System.out.println("");
+	    		System.out.println("Date\t\t                Cal Year\tFiscal Year\tCal Quarter\tFiscal Quarter\tCal Month\tFiscal Month\tCal Week\tFiscal Week");
 		    	for(int i=0;i<365;i++)
 		    	{
-		    		System.out.print(sdf.format(cal.getTime())+"\t");
+		    		System.out.print(sdf.format(cal.getTime())+"\t\t");
+			        System.out.print(getCalendarYear(cal)+"\t\t");
 			        System.out.print(getFiscalYear(getCalendarYear(cal),getCalendarMonth(cal), fiscalMonthOffset, isYearEndFiscalYear)+"\t\t");
-			        System.out.print(getFiscalQuarter(getCalendarMonth(cal),fiscalMonthOffset)+"\t\t"+getCalendarQuarter(getCalendarMonth(cal))+"\t\t");
+			        System.out.print(getCalendarQuarter(getCalendarMonth(cal))+"\t\t");
+			        System.out.print(getFiscalQuarter(getCalendarMonth(cal),fiscalMonthOffset)+"\t\t");
+			        System.out.print((getCalendarMonth(cal)+1)+"\t\t"); 
 			        System.out.print((getFiscalMonth(getCalendarMonth(cal), fiscalMonthOffset)+1)+"\t\t"); 
-			        System.out.print(getFiscalWeek(cal, fiscalMonthOffset, firstDayOfWeek)+"\t\t");
-			        System.out.println(getCalendarWeek(cal.get(Calendar.DAY_OF_YEAR), minimalDaysInFirstWeek));
+			        System.out.print(getCalendarWeek(cal.get(Calendar.DAY_OF_YEAR), minimalDaysInFirstWeek)+"\t\t");
+			        System.out.println(getFiscalWeek(cal, fiscalMonthOffset, firstDayOfWeek));
 		    		cal.add(Calendar.DAY_OF_YEAR, 1);
 		    		cal.getTime();
 		    	}