HugeFileGenerator.java

/** *****************************************************************************
 * Copyright 2014 uniVocity Software Pty Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ***************************************************************************** */
package com.univocity.articles.csvcomparison;

import java.io.*;

import com.univocity.parsers.csv.*;

public class HugeFileGenerator {

    public static void generateHugeFile(File input, String inputEncoding, int timesToExpand, File hugeFile) throws Exception {

        if (hugeFile.exists()) {
            System.out.println("Huge file already generated.");
            return;
        }

        CsvParserSettings readerSettings = new CsvParserSettings();
        readerSettings.getFormat().setLineSeparator("\n");
        CsvParser parser = new CsvParser(readerSettings);

        CsvWriterSettings settings = new CsvWriterSettings();
        settings.setQuoteAllFields(true); //let's see how all parsers perform when the contents are enclosed within quotes.
        settings.getFormat().setLineSeparator("\n");

        CsvWriter writer = new CsvWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(hugeFile), inputEncoding)), settings);
        long totalTime = 0L;
        try {
            Object[] row;
            for (int i = 0; i < timesToExpand; i++) {
                long start = System.currentTimeMillis();

                parser.beginParsing(new BufferedReader(new InputStreamReader(new FileInputStream(input), inputEncoding)));
                while ((row = parser.parseNext()) != null) {
                    writer.writeRow(row);
                }
                long loopTime = System.currentTimeMillis() - start;
                totalTime += loopTime;
                System.out.println("Loop " + (i + 1) + " took " + loopTime + "ms. Total time: " + totalTime + " ms");
            }
        }
        finally {
            writer.close();
        }
        System.out.println("Finished!");
    }

}