Skip to content

Commit

Permalink
Merge "use compaction strategy options" from Raphael
Browse files Browse the repository at this point in the history
  • Loading branch information
@avikivity
avikivity committed Jul 27, 2015
2 parents e24d6c2 + 634d005 commit 2e745be
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 8 deletions.
2 changes: 1 addition & 1 deletion compaction_strategy.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,6 @@ public:
};

// Creates a compaction_strategy object from one of the strategies available.
compaction_strategy make_compaction_strategy(compaction_strategy_type strategy);
compaction_strategy make_compaction_strategy(compaction_strategy_type strategy, const std::map<sstring, sstring>& options);

}
17 changes: 17 additions & 0 deletions cql3/statements/property_definitions.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,10 @@ public:
// Return a property value, typed as a double
double get_double(sstring key, double default_value) const {
auto value = get_simple(key);
return to_double(key, value, default_value);
}

static double to_double(sstring key, std::experimental::optional<sstring> value, double default_value) {
if (value) {
auto val = value.value();
try {
Expand Down Expand Up @@ -167,6 +171,19 @@ public:
return default_value;
}
}

static long to_long(sstring key, std::experimental::optional<sstring> value, long default_value) {
if (value) {
auto val = value.value();
try {
return std::stol(val);
} catch (const std::exception& e) {
throw exceptions::syntax_exception(sprint("Invalid long value %s for '%s'", val, key));
}
} else {
return default_value;
}
}
};

}
Expand Down
2 changes: 1 addition & 1 deletion database.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ void column_family::trigger_compaction() {
}

void column_family::set_compaction_strategy(sstables::compaction_strategy_type strategy) {
_compaction_strategy = make_compaction_strategy(strategy);
_compaction_strategy = make_compaction_strategy(strategy, _schema->compaction_strategy_options());
}

size_t column_family::sstables_count() {
Expand Down
98 changes: 92 additions & 6 deletions sstables/compaction.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include "compaction_strategy.hh"
#include "mutation_reader.hh"
#include "schema.hh"
#include "cql3/statements/property_definitions.hh"

namespace sstables {

Expand Down Expand Up @@ -167,18 +168,99 @@ class major_compaction_strategy : public compaction_strategy_impl {
}
};

class size_tiered_compaction_strategy : public compaction_strategy_impl {
class size_tiered_compaction_strategy_options {
static constexpr uint64_t DEFAULT_MIN_SSTABLE_SIZE = 50L * 1024L * 1024L;
static constexpr double DEFAULT_BUCKET_LOW = 0.5;
static constexpr double DEFAULT_BUCKET_HIGH = 1.5;
static constexpr double DEFAULT_COLD_READS_TO_OMIT = 0.05;
const sstring MIN_SSTABLE_SIZE_KEY = "min_sstable_size";
const sstring BUCKET_LOW_KEY = "bucket_low";
const sstring BUCKET_HIGH_KEY = "bucket_high";
const sstring COLD_READS_TO_OMIT_KEY = "cold_reads_to_omit";

// FIXME: user should be able to configure these values.
uint64_t min_sstable_size = DEFAULT_MIN_SSTABLE_SIZE;
double bucket_low = DEFAULT_BUCKET_LOW;
double bucket_high = DEFAULT_BUCKET_HIGH;
double cold_reads_to_omit = DEFAULT_COLD_READS_TO_OMIT;

static std::experimental::optional<sstring> get_value(const std::map<sstring, sstring>& options, const sstring& name) {
auto it = options.find(name);
if (it == options.end()) {
return std::experimental::nullopt;
}
return it->second;
}
public:
size_tiered_compaction_strategy_options(const std::map<sstring, sstring>& options) {
using namespace cql3::statements;

auto tmp_value = get_value(options, MIN_SSTABLE_SIZE_KEY);
min_sstable_size = property_definitions::to_long(MIN_SSTABLE_SIZE_KEY, tmp_value, DEFAULT_MIN_SSTABLE_SIZE);

tmp_value = get_value(options, BUCKET_LOW_KEY);
bucket_low = property_definitions::to_double(BUCKET_LOW_KEY, tmp_value, DEFAULT_BUCKET_LOW);

tmp_value = get_value(options, BUCKET_HIGH_KEY);
bucket_high = property_definitions::to_double(BUCKET_HIGH_KEY, tmp_value, DEFAULT_BUCKET_HIGH);

tmp_value = get_value(options, COLD_READS_TO_OMIT_KEY);
cold_reads_to_omit = property_definitions::to_double(COLD_READS_TO_OMIT_KEY, tmp_value, DEFAULT_COLD_READS_TO_OMIT);
}

size_tiered_compaction_strategy_options() {
min_sstable_size = DEFAULT_MIN_SSTABLE_SIZE;
bucket_low = DEFAULT_BUCKET_LOW;
bucket_high = DEFAULT_BUCKET_HIGH;
cold_reads_to_omit = DEFAULT_COLD_READS_TO_OMIT;
}

// FIXME: convert java code below.
#if 0
public static Map<String, String> validateOptions(Map<String, String> options, Map<String, String> uncheckedOptions) throws ConfigurationException
{
String optionValue = options.get(MIN_SSTABLE_SIZE_KEY);
try
{
long minSSTableSize = optionValue == null ? DEFAULT_MIN_SSTABLE_SIZE : Long.parseLong(optionValue);
if (minSSTableSize < 0)
{
throw new ConfigurationException(String.format("%s must be non negative: %d", MIN_SSTABLE_SIZE_KEY, minSSTableSize));
}
}
catch (NumberFormatException e)
{
throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, MIN_SSTABLE_SIZE_KEY), e);
}

double bucketLow = parseDouble(options, BUCKET_LOW_KEY, DEFAULT_BUCKET_LOW);
double bucketHigh = parseDouble(options, BUCKET_HIGH_KEY, DEFAULT_BUCKET_HIGH);
if (bucketHigh <= bucketLow)
{
throw new ConfigurationException(String.format("%s value (%s) is less than or equal to the %s value (%s)",
BUCKET_HIGH_KEY, bucketHigh, BUCKET_LOW_KEY, bucketLow));
}

double maxColdReadsRatio = parseDouble(options, COLD_READS_TO_OMIT_KEY, DEFAULT_COLD_READS_TO_OMIT);
if (maxColdReadsRatio < 0.0 || maxColdReadsRatio > 1.0)
{
throw new ConfigurationException(String.format("%s value (%s) should be between between 0.0 and 1.0",
COLD_READS_TO_OMIT_KEY, optionValue));
}

uncheckedOptions.remove(MIN_SSTABLE_SIZE_KEY);
uncheckedOptions.remove(BUCKET_LOW_KEY);
uncheckedOptions.remove(BUCKET_HIGH_KEY);
uncheckedOptions.remove(COLD_READS_TO_OMIT_KEY);

return uncheckedOptions;
}
#endif
friend class size_tiered_compaction_strategy;
};

class size_tiered_compaction_strategy : public compaction_strategy_impl {
size_tiered_compaction_strategy_options _options;

// Return a list of pair of shared_sstable and its respective size.
std::vector<std::pair<sstables::shared_sstable, uint64_t>> create_sstable_and_length_pairs(const sstable_list& sstables);

Expand All @@ -202,6 +284,10 @@ class size_tiered_compaction_strategy : public compaction_strategy_impl {
return n / sstables.size();
}
public:
size_tiered_compaction_strategy() = default;
size_tiered_compaction_strategy(const std::map<sstring, sstring>& options) :
_options(options) {}

virtual future<> compact(column_family& cfs) override;

friend std::vector<sstables::shared_sstable> size_tiered_most_interesting_bucket(lw_shared_ptr<sstable_list>);
Expand Down Expand Up @@ -247,8 +333,8 @@ size_tiered_compaction_strategy::get_buckets(const sstable_list& sstables) {
std::vector<sstables::shared_sstable> bucket = entry.second;
size_t old_average_size = entry.first;

if ((size > (old_average_size * bucket_low) && size < (old_average_size * bucket_high))
|| (size < min_sstable_size && old_average_size < min_sstable_size))
if ((size > (old_average_size * _options.bucket_low) && size < (old_average_size * _options.bucket_high))
|| (size < _options.min_sstable_size && old_average_size < _options.min_sstable_size))
{
size_t total_size = bucket.size() * old_average_size;
size_t new_average_size = (total_size + size) / (bucket.size() + 1);
Expand Down Expand Up @@ -361,7 +447,7 @@ future<> compaction_strategy::compact(column_family& cfs) {
return _compaction_strategy_impl->compact(cfs);
}

compaction_strategy make_compaction_strategy(compaction_strategy_type strategy) {
compaction_strategy make_compaction_strategy(compaction_strategy_type strategy, const std::map<sstring, sstring>& options) {
::shared_ptr<compaction_strategy_impl> impl;

switch(strategy) {
Expand All @@ -372,7 +458,7 @@ compaction_strategy make_compaction_strategy(compaction_strategy_type strategy)
impl = make_shared<major_compaction_strategy>(major_compaction_strategy());
break;
case compaction_strategy_type::size_tiered:
impl = make_shared<size_tiered_compaction_strategy>(size_tiered_compaction_strategy());
impl = make_shared<size_tiered_compaction_strategy>(size_tiered_compaction_strategy(options));
break;
default:
throw std::runtime_error("strategy not supported");
Expand Down

0 comments on commit 2e745be

Please sign in to comment.