IntegrationTestGenerator.java

package net.agkn.hll;

/*
 * Copyright 2013 Aggregate Knowledge, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.FileWriter;
import java.io.IOException;
import java.util.Random;

import net.agkn.hll.HLL;
import net.agkn.hll.HLLType;
import net.agkn.hll.serialization.ISchemaVersion;
import net.agkn.hll.serialization.SerializationUtil;
import net.agkn.hll.util.NumberUtil;
import static net.agkn.hll.ProbabilisticTestUtil.constructHLLValue;

/**
 * Generates test files for testing other implementations of HLL
 * serialization/deserialization, namely the PostgreSQL implementation.
 *
 * @author timon
 */
public class IntegrationTestGenerator {
    // ************************************************************************
    // directory to output the generated tests
    private static final String OUTPUT_DIRECTORY = "/tmp/hll_test/";
    // seed to make results reproducible
    private static final long SEED = 1L;

    // ------------------------------------------------------------------------
    // configurations for HLLs, should mirror settings in PostgreSQL impl. tests
    private static final int REGWIDTH = 5;
    private static final int LOG2M = 11;
    // NOTE:  This differs from the PostgreSQL impl. parameter 'expthresh'. This
    //        is a literal threshold to use in the promotion hierarchy, implying
    //        that both EXPLICIT representation should be used and it should
    //        NOT be automatically computed. This is done to ensure that the
    //        parameters of the test are very explicitly defined.
    private static final int EXPLICIT_THRESHOLD = 256;
    // NOTE:  This is not the PostgreSQL impl. parameter 'sparseon'. 'sparseon'
    //        is assumed to be true and this is a literal register-count threshold
    //        to use in the promotion hierarchy. This is done to ensure that the
    //        parameters of the test are very explicitly defined.
    private static final int SPARSE_THRESHOLD = 850;

    // ------------------------------------------------------------------------
    // computed constants
    private static final int REGISTER_COUNT = (1 << LOG2M);
    private static final int REGISTER_MAX_VALUE = (1 << REGWIDTH) - 1;

    // ========================================================================
    // Tests
    /**
     * Cumulatively adds random values to a FULL HLL through the small range
     * correction, uncorrected range, and large range correction of the HLL's
     * cardinality estimator.
     *
     * Format: cumulative add
     * Tests:
     * - FULL cardinality computation
     */
    private static void fullCardinalityCorrectionTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "cardinality_correction", TestType.ADD);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.FULL);
        initLineAdd(output, hll, schemaVersion);

        // run through some values in the small range correction
        for(int i=0; i<((1 << LOG2M) - 1); i++) {
            final long rawValue = constructHLLValue(LOG2M, i, 1);
            cumulativeAddLine(output, hll, rawValue, schemaVersion);
        }

        // run up past some values in the uncorrected range
        for(int i=0; i<(1 << LOG2M); i++) {
            final long rawValue = constructHLLValue(LOG2M, i, 7);
            cumulativeAddLine(output, hll, rawValue, schemaVersion);
        }

        // run through some values in the large range correction
        for(int i=0; i<(1 << LOG2M); i++) {
            final long rawValue = constructHLLValue(LOG2M, i, 30);
            cumulativeAddLine(output, hll, rawValue, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Cumulatively adds random values to an EMPTY HLL.
     *
     * Format: cumulative add
     * Tests:
     * - EMPTY, EXPLICIT, SPARSE, PROBABILSTIC addition
     * - EMPTY to EXPLICIT promotion
     * - EXPLICIT to SPARSE promotion
     * - SPARSE to FULL promotion
     */
    private static void globalStepTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "comprehensive_promotion", TestType.ADD);

        final Random random = new Random(SEED);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        initLineAdd(output, hll, schemaVersion);

        for(int i=0; i<10000/*arbitrary*/; i++) {
            cumulativeAddLine(output, hll, random.nextLong(), schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Cumulatively unions "underpopulated" FULL HLLs into the
     * accumulator to verify the correct behavior from the PostgreSQL implementation.
     * The PostgreSQL implementation's representations of probabilistic HLLs should
     * depend exclusively on the chosen SPARSE-to-FULL cutoff.
     *
     * Format: cumulative union
     * Tests:
     * - EMPTY U "underpopulated" FULL => SPARSE
     * - SPARSE U "underpopulated" FULL => SPARSE
     * - SPARSE U "barely underpopulated" FULL => FULL
     */
    private static void sparseFullRepresentationTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_full_representation", TestType.UNION);

        final HLL emptyHLL1 = newHLL(HLLType.EMPTY);
        final HLL emptyHLL2 = newHLL(HLLType.EMPTY);

        cumulativeUnionLine(output, emptyHLL1, emptyHLL2, schemaVersion);

        // NOTE:  In this test the sparseReference will be the "expected" value
        //        from the C representation, since it doesn't choose representation
        //        based on original encoding, but rather on the promotion rules
        //        and the declared type of the "receiving" field.
        //        It is the manually-constructed union result.

        // "underpopulated" FULL U EMPTY => SPARSE
        final HLL fullHLL = newHLL(HLLType.FULL);
        fullHLL.addRaw(constructHLLValue(LOG2M, 0/*ix*/, 1/*val*/));

        final HLL sparseHLL = newHLL(HLLType.SPARSE);
        sparseHLL.addRaw(constructHLLValue(LOG2M, 0/*ix*/, 1/*val*/));

        output.write(stringCardinality(fullHLL) + "," + toByteA(fullHLL, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
        output.flush();

        // "underpopulated" FULL (small) U SPARSE (small) => SPARSE
        final HLL fullHLL2 = newHLL(HLLType.FULL);
        fullHLL2.addRaw(constructHLLValue(LOG2M, 1/*ix*/, 1/*val*/));

        sparseHLL.addRaw(constructHLLValue(LOG2M, 1/*ix*/, 1/*val*/));

        output.write(stringCardinality(fullHLL2) + "," + toByteA(fullHLL2, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
        output.flush();

        // "underpopulated" FULL (just on edge) U SPARSE (small) => FULL
        final HLL fullHLL3 = newHLL(HLLType.FULL);
        for(int i=2; i<(SPARSE_THRESHOLD + 1); i++) {
            fullHLL3.addRaw(constructHLLValue(LOG2M, i/*ix*/, 1/*val*/));
            sparseHLL.addRaw(constructHLLValue(LOG2M, i/*ix*/, 1/*val*/));
        }

        output.write(stringCardinality(fullHLL3) + "," + toByteA(fullHLL3, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
        output.flush();
    }

    /**
     * Cumulatively sets successive registers to:
     *
     *     <code>(registerIndex % REGISTER_MAX_VALUE) + 1</code>
     *
     * by adding specifically constructed values to a SPARSE HLL.
     * Does not induce promotion.
     *
     * Format: cumulative add
     * Tests:
     * - SPARSE addition (predictable)
     */
    private static void sparseStepTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_step", TestType.ADD);

        // the accumulator, starts empty sparse probabilistic
        final HLL hll = newHLL(HLLType.SPARSE);
        initLineAdd(output, hll, schemaVersion);

        for(int i=0; i<SPARSE_THRESHOLD; i++) {
            final long rawValue = constructHLLValue(LOG2M, i, ((i % REGISTER_MAX_VALUE) + 1));
            cumulativeAddLine(output, hll, rawValue, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Cumulatively sets random registers of a SPARSE HLL to
     * random values by adding random values. Does not induce promotion.
     *
     * Format: cumulative add
     * Tests:
     * - SPARSE addition (random)
     */
    private static void sparseRandomTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_random", TestType.ADD);

        final Random random = new Random(SEED);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.SPARSE);
        initLineAdd(output, hll, schemaVersion);

        for(int i=0; i<SPARSE_THRESHOLD; i++) {
            final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
            final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
            final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);

            cumulativeAddLine(output, hll, rawValue, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Cumulatively sets the first register (index 0) to value 2, the last
     * register (index m-1) to value 2, and then sets registers with indices in
     * the range 2 to (sparseCutoff + 2) to value 1 to trigger promotion.
     *
     * This tests for register alignment in the promotion from SPARSE
     * to FULL.
     *
     * Format: cumulative add
     * Tests:
     * - SPARSE addition
     * - SPARSE to FULL promotion
     */
    private static void sparseEdgeTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_edge", TestType.ADD);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.SPARSE);
        initLineAdd(output, hll, schemaVersion);

        final long firstValue = constructHLLValue(LOG2M, 0, 2);
        cumulativeAddLine(output, hll, firstValue, schemaVersion);

        final long lastValue = constructHLLValue(LOG2M, (1 << LOG2M) - 1, 2);
        cumulativeAddLine(output, hll, lastValue, schemaVersion);

        for(int i=2; i<(SPARSE_THRESHOLD + 2); i++) {
            final long middleValue = constructHLLValue(LOG2M, i, 1);

            cumulativeAddLine(output, hll, middleValue, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Unions an EMPTY accumulator with EXPLICIT HLLs, each containing a
     * single random value.
     *
     * Format: cumulative union
     * Tests:
     * - EMPTY U EXPLICIT
     * - EXPLICIT U EXPLICIT
     * - EXPLICIT to SPARSE promotion
     * - SPARSE U EXPLICIT
     */
    private static void explicitPromotionTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "explicit_promotion", TestType.UNION);

        final Random random = new Random(SEED);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);
        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

        for(int i=0; i<(EXPLICIT_THRESHOLD+500)/*should be greater than promotion cutoff*/; i++) {
            // make an EXPLICIT set and populate with cardinality 1
            final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
            explicitHLL.addRaw(random.nextLong());

            cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Unions an EMPTY accumulator with SPARSE HLLs, each
     * having one register set.
     *
     * Format: cumulative union
     * Tests:
     * - EMPTY U SPARSE
     * - SPARSE U SPARSE
     * - SPARSE promotion
     * - SPARSE U FULL
     */
    private static void sparseProbabilisticPromotionTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_promotion", TestType.UNION);

        final Random random = new Random(SEED);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);
        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);


        for(int i=0; i<(SPARSE_THRESHOLD + 1000)/*should be greater than promotion cutoff*/; i++) {
            // make a SPARSE set and populate with cardinality 1
            final HLL sparseHLL = newHLL(HLLType.SPARSE);

            final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
            final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
            final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
            sparseHLL.addRaw(rawValue);

            cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Unions an EMPTY accumulator with EXPLICIT HLLs, each having a single
     * random value, twice in a row to verify that the set properties are
     * satisfied.
     *
     * Format: cumulative union
     * Tests:
     * - EMPTY U EXPLICIT
     * - EXPLICIT U EXPLICIT
     */
    private static void explicitOverlapTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "explicit_explicit", TestType.UNION);

        final Random random = new Random(SEED);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);

        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

        for(int i=0; i<EXPLICIT_THRESHOLD; i++) {
            // make an EXPLICIT set and populate with cardinality 1
            final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
            explicitHLL.addRaw(random.nextLong());

            // union it into the accumulator twice, to test overlap (cardinality should not change)
            cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
            cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Unions an EMPTY accumulator with SPARSE HLLs, each
     * having a single register set, twice in a row to verify that the set
     * properties are satisfied.
     *
     * Format: cumulative union
     * Tests:
     * - EMPTY U SPARSE
     * - SPARSE U SPARSE
     */
    private static void sparseProbabilisticOverlapTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_sparse", TestType.UNION);

        final Random random = new Random(SEED);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);

        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

        for(int i=0; i<SPARSE_THRESHOLD; i++) {
            // make a SPARSE set and populate with cardinality 1
            final HLL sparseHLL = newHLL(HLLType.SPARSE);
            final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
            final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
            final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
            sparseHLL.addRaw(rawValue);

            cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Unions an EMPTY accumulator with FULL HLLs, each having
     * many registers set, twice in a row to verify that the set properties are
     * satisfied.
     *
     * Format: cumulative union
     * Tests:
     * - EMPTY U FULL
     * - FULL U FULL
     */
    private static void probabilisticUnionTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "probabilistic_probabilistic", TestType.UNION);

        final Random random = new Random(SEED);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);
        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

        for(int i=0; i<1000/*number of rows to generate*/; i++) {
            // make a FULL set and populate with
            final HLL fullHLL = newHLL(HLLType.FULL);
            final int elementCount = random.nextInt(10000/*arbitrary maximum cardinality*/);
            for(int j=0;j<elementCount;j++) {
                fullHLL.addRaw(random.nextLong());
            }

            cumulativeUnionLine(output, hll, fullHLL, schemaVersion);
        }

        output.flush();
        output.close();
    }

    /**
     * Unions an EMPTY accumulator with random HLLs.
     *
     * Format: cumulative union
     * Tests:
     * - hopefully all union possibilities
     */
    private static void globalUnionTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "comprehensive", TestType.UNION);

        final Random random = new Random(SEED);

        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);

        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

        for(int i=0; i<1000/*number of rows to generate*/; i++) {
            final HLL randomHLL = generateRandomHLL(random);
            cumulativeUnionLine(output, hll, randomHLL, schemaVersion);
        }

        output.flush();
        output.close();
    }

    // ========================================================================
    // Main
    public static void fullSuite(final ISchemaVersion schemaVersion) throws IOException {
        fullCardinalityCorrectionTest(schemaVersion);
        globalUnionTest(schemaVersion);
        globalStepTest(schemaVersion);
        probabilisticUnionTest(schemaVersion);
        explicitPromotionTest(schemaVersion);
        explicitOverlapTest(schemaVersion);
        sparseFullRepresentationTest(schemaVersion);
        sparseStepTest(schemaVersion);
        sparseRandomTest(schemaVersion);
        sparseEdgeTest(schemaVersion);
        sparseProbabilisticPromotionTest(schemaVersion);
        sparseProbabilisticOverlapTest(schemaVersion);
    }

    public static void main(String[] args) throws IOException {
        fullSuite(SerializationUtil.VERSION_ONE);
    }

    // ************************************************************************
    // Helpers
    /**
     * Shortcut for testing constructor, which uses the constants defined at
     * the top of the file as default parameters.
     *
     * @return a new {@link HLL} of specified type, which uses the parameters
     *         ({@link #LOG2M}, {@link #REGWIDTH}, {@link #EXPLICIT_THRESHOLD},
     *         and {@link #SPARSE_THRESHOLD}) specified above.
     */
    private static HLL newHLL(final HLLType type) {
        return newHLL(type);
    }

    /**
     * Returns the algorithm-specific cardinality of the specified {@link HLL}
     * as a {@link String} appropriate for comparison with the algorithm-specific
     * cardinality provided by the PostgreSQL implementation.
     *
     * @param  hll the HLL whose algorithm-specific cardinality is to be printed.
     *         This cannot be <code>null</code>.
     * @return the algorithm-specific cardinality of the instance as a PostgreSQL-
     *         compatible String. This will never be <code>null</code>
     */
    private static String stringCardinality(final HLL hll) {
        switch(hll.getType()) {
            case EMPTY:
                return "0";
            case EXPLICIT:/*promotion has not yet occurred*/
                return Long.toString(hll.cardinality());
            case SPARSE:
                return Double.toString(hll.sparseProbabilisticAlgorithmCardinality());
            case FULL:
                return Double.toString(hll.fullProbabilisticAlgorithmCardinality());
            default:
                throw new RuntimeException("Unknown HLL type " + hll.getType());
        }
    }

    /**
     * Generates a random HLL and populates it with random values.
     *
     * @param  random the {@link Random random number generator} used to populate
     *         the HLL. This cannot be <code>null</code>.
     * @return the populated HLL. This will never be <code>null</code>.
     */
    public static HLL generateRandomHLL(final Random random) {
        final int randomTypeInt = random.nextInt(HLLType.values().length);
        final HLLType type;
        switch(randomTypeInt) {
            case 0:
                type = HLLType.EMPTY;
                break;
            case 1:
                type = HLLType.EXPLICIT;
                break;
            case 2:
                type = HLLType.FULL;
                break;
            case 3:
                type = HLLType.EMPTY;
                break;
            case 4:
                type = HLLType.SPARSE;
                break;
            default:
                throw new RuntimeException("Unassigned type int " + randomTypeInt);
        }

        final int cardinalityCap;
        final int cardinalityBaseline;

        switch(type) {
            case EMPTY:
                return newHLL(HLLType.EMPTY);
            case EXPLICIT:
                cardinalityCap = EXPLICIT_THRESHOLD;
                cardinalityBaseline = 1;
                break;
            case SPARSE:
                cardinalityCap = SPARSE_THRESHOLD;
                cardinalityBaseline = (EXPLICIT_THRESHOLD + 1);
                break;
            case FULL:
                cardinalityCap = 100000;
                cardinalityBaseline = (SPARSE_THRESHOLD*10);
                break;
            default:
                throw new RuntimeException("We should never be here.");
        }

        final HLL hll = newHLL(HLLType.EMPTY);
        for(int i=0; i<cardinalityBaseline; i++) {
            hll.addRaw(random.nextLong());
        }
        for(int i=0; i<random.nextInt(cardinalityCap - cardinalityBaseline); i++) {
            hll.addRaw(random.nextLong());
        }

        return hll;
    }

    /**
     * Opens a {@link FileWriter} and writes out an appropriate CSV header.
     *
     * @param  schemaVersion Schema version of the output. This cannot be
     *         <code>null</code>.
     * @param  description Description string used to build the filename.
     *         This cannot be <code>null</code>.
     * @param  type {@link TestType type} of the test file to be written.
     *         This cannot be <code>null</code>.
     * @return The opened {@link FileWriter writer}. This will never be <code>null</code>.
     */
    private static FileWriter openOutput(final ISchemaVersion schemaVersion, final String description, final TestType type) throws IOException {
        final String schemaVersionPrefix = "v"+ schemaVersion.schemaVersionNumber() + "_";
        final String header;
        final String filename;
        switch(type) {
            case ADD:
                header = "cardinality,raw_value,HLL\n";
                filename = schemaVersionPrefix + "cumulative_add_" + description + ".csv";
                break;
            case UNION:
                header = "cardinality,HLL,union_cardinality,union_HLL\n";
                filename = schemaVersionPrefix + "cumulative_union_" + description + ".csv";
                break;
            default:
                throw new RuntimeException("Unknown test type " + type);
        }

        final FileWriter output = new FileWriter(OUTPUT_DIRECTORY + filename);
        output.write(header);
        output.flush();
        return output;
    }

    /**
     * Writes out a {@link TestType#ADD}-formatted test line.
     *
     * @param  output The output {@link FileWriter writer}. This cannot be <code>null</code>.
     * @param  hll The "accumulator" HLL instance. This cannot be <code>null</code>.
     * @param  rawValue The raw value added to the HLL.
     * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
     *         be <code>null</code>.
     */
    private static void cumulativeAddLine(final FileWriter output, final HLL hll, final long rawValue, final ISchemaVersion schemaVersion) throws IOException {
        hll.addRaw(rawValue);
        final String accumulatorCardinality = stringCardinality(hll);

        output.write(accumulatorCardinality + "," + rawValue + "," + toByteA(hll, schemaVersion) + "\n");
        output.flush();
    }

    /**
     * Writes an initial line for a {@link TestType#ADD}-formatted test.
     *
     * @param  output The output {@link FileWriter writer}. This cannot be <code>null</code>.
     * @param  hll The "accumulator" HLL instance. This cannot be <code>null</code>.
     * @param  rawValue The raw value added to the HLL.
     * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
     *         be <code>null</code>.
     */
    private static void initLineAdd(final FileWriter output, final HLL hll, final ISchemaVersion schemaVersion) throws IOException {
        output.write(0 + "," + 0 + "," + toByteA(hll, schemaVersion) + "\n");
        output.flush();
    }

    /**
     * Writes out a {@link TestType#UNION}-formatted test line.
     *
     * @param  output The output {@link FileWriter writer}. This cannot be <code>null</code>.
     * @param  hll The "accumulator" HLL instance. This cannot be <code>null</code>.
     * @param  increment The "increment" HLL instance which will be unioned into
     *         the accumulator. This cannot be <code>null</code>.
     * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
     *         be <code>null</code>.
     */
    private static void cumulativeUnionLine(final FileWriter output, final HLL hll, final HLL increment, final ISchemaVersion schemaVersion) throws IOException {
        hll.union(increment);

        final String incrementCardinality = stringCardinality(increment);
        final String accumulatorCardinality = stringCardinality(hll);
        output.write(incrementCardinality + "," + toByteA(increment, schemaVersion) + "," + accumulatorCardinality + "," + toByteA(hll, schemaVersion) + "\n");
        output.flush();
    }

    /**
     * Serializes a HLL to Postgres 9 'bytea' hex-format, for CSV ingest.
     *
     * @param  hll the HLL to serialize. This cannot be <code>null</code>.
     * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
     *         be <code>null</code>.
     * @return a PostgreSQL 'bytea' string representing the HLL.
     */
    private static String toByteA(final HLL hll, final ISchemaVersion schemaVersion) {
        final byte[] bytes = hll.toBytes(schemaVersion);
        return ("\\x" + NumberUtil.toHex(bytes, 0, bytes.length));
    }

    /**
     * Indicates what kind of test output a test will generate.
     */
    private static enum TestType {
        /**
         * This type of test is characterized by values being added to an
         * accumulator HLL whose serialized representation (after the value is added)
         * is printed to each line along with the cardinality and added value.
         */
        ADD,
        /**
         * This type of test is characterized by HLLs being unioned into an
         * accumulator HLL whose serialized representation (after the HLL is
         * union'd) is printed to each line along with the cardinalities and the
         * serialized representation of the HLL union'd in.
         */
        UNION;
    }
}