Skip to content
Permalink
Browse files
Core improvements 
    Major features:
      * Implement updates
      * Make sure we can advance RNGs from zero as well
      * Fix a problem with predictable descriptor
    Bugfixes:
      * Fix column mask inconsistencies
      * Fix a problem with partition key liveness info
    Quality of life improvements:
      * Get rid of driver dependency for query generation
      * Get rid of guava dependency
      * Add reusable config files
      * Switch from streams to iterables
    General improvements:
      * Make unset and nil descriptors more distinct and harder to generate particularly for the smaller descriptors
      * Fixed schema configurator to allow empty column sets
      * Move workloads to a common dir
      * Fixed schema configurator to output correct json
      * No-op checker to execute with Quorum, not ALL
      * Make tag for build unique

Patch by Alex Petrov for CASSANDRA-16262
  • Loading branch information
@ifesdjeen
ifesdjeen committed Oct 1, 2021
1 parent ddd643e commit df40fa39ff1cee5165257a8334eee1ce78f829fe
Showing 32 changed files with 690 additions and 357 deletions.
9 harry-core/pom.xml
View file
@@ -33,10 +33,11 @@
<name>Harry Core</name>

<dependencies>
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-core</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.25</version>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
44 harry-core/src/harry/core/Configuration.java
View file
@@ -38,10 +38,13 @@
import harry.ddl.SchemaSpec;
import harry.generators.Surjections;
import harry.generators.distribution.Distribution;
import harry.model.AlwaysSamePartitionSelector;
import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.QuiescentChecker;
import harry.model.clock.ApproximateMonotonicClock;
import harry.model.clock.OffsetClock;
import harry.model.sut.PrintlnSut;
import harry.model.sut.SystemUnderTest;
import harry.runner.AllPartitionsValidator;
import harry.runner.CorruptingPartitionVisitor;
@@ -92,7 +95,10 @@
mapper.registerSubtypes(CorruptingPartitionVisitorConfiguration.class);
mapper.registerSubtypes(RecentPartitionsValidatorConfiguration.class);
mapper.registerSubtypes(FixedSchemaProviderConfiguration.class);

mapper.registerSubtypes(AlwaysSamePartitionSelector.AlwaysSamePartitionSelectorConfiguration.class);
mapper.registerSubtypes(OffsetClock.OffsetClockConfiguration.class);
mapper.registerSubtypes(PrintlnSut.PrintlnSutConfiguration.class);
mapper.registerSubtypes(NoOpDataTrackerConfiguration.class);
mapper.registerSubtypes(NoOpMetricReporterConfiguration.class);
}

@@ -409,7 +415,7 @@ public interface DataTrackerConfiguration extends DataTracker.DataTrackerFactory

}

@JsonTypeName("no_op_tracker")
@JsonTypeName("no_op")
public static class NoOpDataTrackerConfiguration implements DataTrackerConfiguration
{
@JsonCreator
@@ -615,7 +621,7 @@ public Model make(Run run)
}
}

@JsonTypeName("no_op_checker")
@JsonTypeName("no_op")
public static class NoOpCheckerConfig implements ModelConfiguration
{
@JsonCreator
@@ -694,7 +700,7 @@ public static class OperationKindSelectorBuilder extends WeightedSelectorBuilder
private Map<OpSelectors.OperationKind, Integer> operation_kind_weights = new OperationKindSelectorBuilder()
.addWeight(OpSelectors.OperationKind.DELETE_ROW, 1)
.addWeight(OpSelectors.OperationKind.DELETE_COLUMN, 1)
.addWeight(OpSelectors.OperationKind.WRITE, 98)
.addWeight(OpSelectors.OperationKind.INSERT, 98)
.build();
private Map<OpSelectors.OperationKind, long[]> column_mask_bitsets;
private int[] fractions;
@@ -1059,6 +1065,12 @@ public SchemaSpec make(long seed)
@JsonTypeName("fixed")
public static class FixedSchemaProviderConfiguration implements SchemaProviderConfiguration
{
public final String keyspace;
public final String table;
public final Map<String, String> partition_keys;
public final Map<String, String> clustering_keys;
public final Map<String, String> regular_columns;
public final Map<String, String> static_keys;
private final SchemaSpec schemaSpec;

@JsonCreator
@@ -1069,10 +1081,28 @@ public FixedSchemaProviderConfiguration(@JsonProperty("keyspace") String keyspac
@JsonProperty("regular_columns") Map<String, String> regulars,
@JsonProperty("static_columns") Map<String, String> statics)
{
this.schemaSpec = SchemaGenerators.parse(keyspace, table,
pks, cks, regulars, statics);
this(SchemaGenerators.parse(keyspace, table,
pks, cks, regulars, statics),
pks,
cks,
regulars,
statics);
}

public FixedSchemaProviderConfiguration(SchemaSpec schemaSpec,
Map<String, String> pks,
Map<String, String> cks,
Map<String, String> regulars,
Map<String, String> statics)
{
this.schemaSpec = schemaSpec;
this.keyspace = schemaSpec.keyspace;
this.table = schemaSpec.table;
this.partition_keys = pks;
this.clustering_keys = cks;
this.regular_columns = regulars;
this.static_keys = statics;
}
public SchemaSpec make(long seed)
{
return schemaSpec;
@@ -1084,7 +1114,7 @@ public interface MetricReporterConfiguration extends MetricReporter.MetricReport
{
}

@JsonTypeName("default")
@JsonTypeName("no_op")
public static class NoOpMetricReporterConfiguration implements MetricReporterConfiguration
{
public MetricReporter make()
5 harry-core/src/harry/core/VisibleForTesting.java
View file
@@ -0,0 +1,5 @@
package harry.core;

public @interface VisibleForTesting {
}

2 harry-core/src/harry/corruptor/RowCorruptor.java
View file
@@ -29,7 +29,7 @@

public interface RowCorruptor
{
Logger logger = LoggerFactory.getLogger(QueryResponseCorruptor.class);
final Logger logger = LoggerFactory.getLogger(QueryResponseCorruptor.class);

boolean canCorrupt(ResultSetRow row);

10 harry-core/src/harry/ddl/ColumnSpec.java
View file
@@ -18,20 +18,18 @@

package harry.ddl;

import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;

import com.google.common.collect.ImmutableList;

import harry.generators.Bijections;
import harry.generators.StringBijection;

import static harry.generators.StringBijection.getByte;

public class ColumnSpec<T>
{
public final String name;
@@ -318,7 +316,7 @@ public int compareLexicographically(long l, long r)
}
};

public static final Collection<DataType<?>> DATA_TYPES = ImmutableList.of(
public static final Collection<DataType<?>> DATA_TYPES = Collections.unmodifiableList(Arrays.asList(
ColumnSpec.int8Type,
ColumnSpec.int16Type,
ColumnSpec.int32Type,
@@ -328,7 +326,7 @@ public int compareLexicographically(long l, long r)
ColumnSpec.doubleType,
ColumnSpec.asciiType,
ColumnSpec.uuidType,
ColumnSpec.timestampType);
ColumnSpec.timestampType));

public static class ReversedType<T> extends DataType<T>
{
76 harry-core/src/harry/ddl/SchemaGenerators.java
View file
@@ -21,16 +21,14 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Supplier;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;

import com.fasterxml.jackson.annotation.JsonProperty;
import harry.generators.Generator;
import harry.generators.Surjections;

@@ -43,32 +41,30 @@ public static Builder schema(String ks)
return new Builder(ks);
}

public static final Collection<ColumnSpec.DataType<?>> clusteringKeyTypes;
public static final Map<String, ColumnSpec.DataType<?>> nameToTypeMap;
public static final Collection<ColumnSpec.DataType<?>> columnTypes;
public static final Collection<ColumnSpec.DataType<?>> partitionKeyTypes;
public static final Collection<ColumnSpec.DataType<?>> clusteringKeyTypes;

static
{
partitionKeyTypes = Collections.unmodifiableList(Arrays.asList(ColumnSpec.int64Type,
ColumnSpec.asciiType,
ColumnSpec.asciiType(4, 5),
ColumnSpec.asciiType(4, 10)));

columnTypes = Collections.unmodifiableList(Arrays.asList(
// ColumnSpec.int8Type,
// ColumnSpec.int16Type,
// ColumnSpec.int32Type,
ColumnSpec.int64Type,
ColumnSpec.asciiType,
ColumnSpec.asciiType(4, 256),
ColumnSpec.asciiType(4, 512)));


ImmutableList.Builder<ColumnSpec.DataType<?>> builder = ImmutableList.builder();
builder.add(ColumnSpec.int8Type,
ColumnSpec.int16Type,
ColumnSpec.int32Type,
ColumnSpec.int64Type,
ColumnSpec.asciiType,
ColumnSpec.asciiType(4, 256),
ColumnSpec.asciiType(4, 512));

columnTypes = builder.build();
builder.add(ColumnSpec.int8Type,
ColumnSpec.int16Type,
ColumnSpec.int32Type,
ColumnSpec.int64Type,
ColumnSpec.asciiType);
builder = ImmutableList.builder();
builder.addAll(columnTypes);

ImmutableMap.Builder<String, ColumnSpec.DataType<?>> mapBuilder = ImmutableMap.builder();
List<ColumnSpec.DataType<?>> builder = new ArrayList<>(columnTypes);
Map<String, ColumnSpec.DataType<?>> mapBuilder = new HashMap<>();

for (ColumnSpec.DataType<?> columnType : columnTypes)
{
@@ -82,8 +78,8 @@ public static Builder schema(String ks)
builder.add(ColumnSpec.floatType);
builder.add(ColumnSpec.doubleType);

clusteringKeyTypes = builder.build();
nameToTypeMap = mapBuilder.build();
clusteringKeyTypes = Collections.unmodifiableList(builder);
nameToTypeMap = Collections.unmodifiableMap(mapBuilder);
}

@SuppressWarnings("unchecked")
@@ -126,7 +122,7 @@ public static Generator<ColumnSpec<?>> columnSpecGenerator(Collection<ColumnSpec

public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
{
return new ColumnSpec<>(prefix + (counter++),
return new ColumnSpec<>(String.format("%s%04d", prefix, counter++),
type,
kind);
}
@@ -143,7 +139,24 @@ public static Generator<ColumnSpec<?>> clusteringColumnSpecGenerator(String pref

public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
{
return ColumnSpec.ck(prefix + (counter++), type);
return ColumnSpec.ck(String.format("%s%04d", prefix, counter++), type);
}
});
}

@SuppressWarnings("unchecked")
public static Generator<ColumnSpec<?>> partitionColumnSpecGenerator(String prefix)
{
return fromValues(partitionKeyTypes)
.map(new Function<ColumnSpec.DataType<?>, ColumnSpec<?>>()
{
private int counter = 0;

public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
{

return ColumnSpec.pk(String.format("%s%04d", prefix, counter++),
type);
}
});
}
@@ -155,10 +168,10 @@ public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
private final String keyspace;
private final Supplier<String> tableNameSupplier;

private Generator<ColumnSpec<?>> pkGenerator = columnSpecGenerator("pk", ColumnSpec.Kind.PARTITION_KEY);
private Generator<ColumnSpec<?>> pkGenerator = partitionColumnSpecGenerator("pk");
private Generator<ColumnSpec<?>> ckGenerator = clusteringColumnSpecGenerator("ck");
private Generator<ColumnSpec<?>> regularGenerator = columnSpecGenerator("regular", ColumnSpec.Kind.REGULAR);
private Generator<ColumnSpec<?>> staticGenerator = columnSpecGenerator("regular", ColumnSpec.Kind.STATIC);
private Generator<ColumnSpec<?>> staticGenerator = columnSpecGenerator("static", ColumnSpec.Kind.STATIC);

private int minPks = 1;
private int maxPks = 1;
@@ -457,6 +470,9 @@ public SchemaSpec get()

public static List<ColumnSpec<?>> toColumns(Map<String, String> config, ColumnSpec.Kind kind, boolean allowReverse)
{
if (config == null)
return Collections.EMPTY_LIST;

List<ColumnSpec<?>> columns = new ArrayList<>(config.size());

for (Map.Entry<String, String> e : config.entrySet())

0 comments on commit df40fa3

Please sign in to comment.