Skip to content

Commit

Permalink
Allow more options for inputting capacities and block sizes in config…
Browse files Browse the repository at this point in the history 
… file
  • Loading branch information
@ChristopherHogan
ChristopherHogan committed Dec 10, 2021
1 parent 68850c7 commit 69a0390
Show file tree
Hide file tree
Showing 3 changed files with 191 additions and 28 deletions.
100 changes: 86 additions & 14 deletions src/config_parser.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,14 @@ static const char *kConfigVariableStrings[ConfigVariable_Count] = {
"unknown",
"num_devices",
"num_targets",
"capacities_bytes",
"capacities_kb",
"capacities_mb",
"capacities_gb",
"block_sizes_bytes",
"block_sizes_kb",
"block_sizes_mb",
"block_sizes_gb",
"num_slabs",
"slab_unit_sizes",
"desired_slab_percentages",
Expand Down Expand Up @@ -689,7 +695,59 @@ void CheckConstraints(Config *config) {
}
}

void RequireCapacitiesUnset(bool &already_specified) {
if (already_specified) {
LOG(FATAL) << "Capacities are specified multiple times in the configuration"
<< " file. Only use one of 'capacities_bytes', 'capacities_kb',"
<< "'capacities_mb', or 'capacities_gb'\n";
} else {
already_specified = true;
}
}

void RequireBlockSizesUnset(bool &already_specified) {
if (already_specified) {
LOG(FATAL) << "Block sizes are specified multiple times in the "
<< "configuration file. Only use one of 'block_sizes_bytes',"
<< "'block_sizes_kb', 'block_sizes_mb', or 'block_sizes_gb'\n";
} else {
already_specified = true;
}
}

Token *ParseCapacities(Config *config, Token *tok, bool &already_specified,
size_t conversion) {
RequireNumDevices(config);
RequireCapacitiesUnset(already_specified);
tok = ParseSizetList(tok, config->capacities, config->num_devices);
for (int i = 0; i < config->num_devices; ++i) {
config->capacities[i] *= conversion;
}

return tok;
}

Token *ParseBlockSizes(Config *config, Token *tok, bool &already_specified,
size_t conversion) {
RequireNumDevices(config);
RequireBlockSizesUnset(already_specified);
tok = ParseIntList(tok, config->block_sizes, config->num_devices);
for (int i = 0; i < config->num_devices; ++i) {
size_t val = (size_t)config->block_sizes[i] * conversion;
if (val > INT_MAX) {
LOG(FATAL) << "Max supported block size is " << INT_MAX << " bytes. "
<< "Config file requested " << val << " bytes\n";
}
config->block_sizes[i] *= conversion;
}

return tok;
}

void ParseTokens(TokenList *tokens, Config *config) {
bool capacities_specified = false;
bool block_sizes_specified = false;

Token *tok = tokens->head;
while (tok) {
ConfigVariable var = GetConfigVariable(tok);
Expand Down Expand Up @@ -719,22 +777,36 @@ void ParseTokens(TokenList *tokens, Config *config) {
config->num_targets = val;
break;
}
case ConfigVariable_Capacities: {
RequireNumDevices(config);
tok = ParseSizetList(tok, config->capacities, config->num_devices);
// NOTE(chogan): Convert from MB to bytes
for (int i = 0; i < config->num_devices; ++i) {
config->capacities[i] *= 1024 * 1024;
}
case ConfigVariable_CapacitiesB: {
tok = ParseCapacities(config, tok, capacities_specified, 1);
break;
}
case ConfigVariable_BlockSizes: {
RequireNumDevices(config);
tok = ParseIntList(tok, config->block_sizes, config->num_devices);
// NOTE(chogan): Convert from KB to bytes
for (int i = 0; i < config->num_devices; ++i) {
config->block_sizes[i] *= 1024;
}
case ConfigVariable_CapacitiesKb: {
tok = ParseCapacities(config, tok, capacities_specified, KILOBYTES(1));
break;
}
case ConfigVariable_CapacitiesMb: {
tok = ParseCapacities(config, tok, capacities_specified, MEGABYTES(1));
break;
}
case ConfigVariable_CapacitiesGb: {
tok = ParseCapacities(config, tok, capacities_specified, GIGABYTES(1));
break;
}
case ConfigVariable_BlockSizesB: {
tok = ParseBlockSizes(config, tok, block_sizes_specified, 1);
break;
}
case ConfigVariable_BlockSizesKb: {
tok = ParseBlockSizes(config, tok, block_sizes_specified, KILOBYTES(1));
break;
}
case ConfigVariable_BlockSizesMb: {
tok = ParseBlockSizes(config, tok, block_sizes_specified, MEGABYTES(1));
break;
}
case ConfigVariable_BlockSizesGb: {
tok = ParseBlockSizes(config, tok, block_sizes_specified, GIGABYTES(1));
break;
}
case ConfigVariable_NumSlabs: {
Expand Down
10 changes: 8 additions & 2 deletions src/config_parser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,14 @@ enum ConfigVariable {
ConfigVariable_Unkown,
ConfigVariable_NumDevices,
ConfigVariable_NumTargets,
ConfigVariable_Capacities,
ConfigVariable_BlockSizes,
ConfigVariable_CapacitiesB,
ConfigVariable_CapacitiesKb,
ConfigVariable_CapacitiesMb,
ConfigVariable_CapacitiesGb,
ConfigVariable_BlockSizesB,
ConfigVariable_BlockSizesKb,
ConfigVariable_BlockSizesMb,
ConfigVariable_BlockSizesGb,
ConfigVariable_NumSlabs,
ConfigVariable_SlabUnitSizes,
ConfigVariable_DesiredSlabPercentages,
Expand Down
109 changes: 97 additions & 12 deletions test/config_parser_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ using hermes::Arena;
using hermes::u8;
using hermes::Config;

namespace hermes {
namespace testing {

Config ParseConfigString(Arena *arena, const std::string &config_string) {
hermes::ScopedTemporaryMemory scratch(arena);
hermes::EntireFile config_file =
Expand Down Expand Up @@ -71,20 +74,86 @@ void TestParseRangeList(Arena *arena) {
}
}

int main(int argc, char **argv) {
if (argc < 2) {
fprintf(stderr, "Expected a path to a hermes.conf file\n");
exit(-1);
void RunCapacityValuesTest(Arena *arena, const std::string &config_string,
const std::vector<size_t> &expected) {
Config config = ParseConfigString(arena, config_string);
Assert((size_t)config.num_devices == expected.size());
for (int i = 0; i < config.num_devices; ++i) {
Assert(config.capacities[i] == expected[i]);
}
}

hermes::Config config = {};
void TestCapacityValues(Arena *arena) {
std::string base_config = "num_devices = 4;\n";

const size_t kConfigMemorySize = KILOBYTES(16);
u8 config_memory[kConfigMemorySize];
Arena arena = {};
hermes::InitArena(&arena, kConfigMemorySize, config_memory);
{
std::vector<size_t> expected{50, 50, 50, 50};
std::string config_string = "capacities_bytes = {50, 50, 50, 50};\n";
RunCapacityValuesTest(arena, base_config + config_string, expected);
}

{
std::vector<size_t> expected{KILOBYTES(50), KILOBYTES(50), KILOBYTES(50),
KILOBYTES(50)};
std::string config_string = "capacities_kb = {50, 50, 50, 50};\n";
RunCapacityValuesTest(arena, base_config + config_string, expected);
}

{
std::vector<size_t> expected{MEGABYTES(50), MEGABYTES(50), MEGABYTES(50),
MEGABYTES(50)};
std::string config_string = "capacities_mb = {50, 50, 50, 50};\n";
RunCapacityValuesTest(arena, base_config + config_string, expected);
}

{
std::vector<size_t> expected{GIGABYTES(50), GIGABYTES(50), GIGABYTES(50),
GIGABYTES(50)};
std::string config_string = "capacities_gb = {50, 50, 50, 50};\n";
RunCapacityValuesTest(arena, base_config + config_string, expected);
}
}

void RunBlockSizesTest(Arena *arena, const std::string &config_string,
const std::vector<int> &expected) {
Config config = ParseConfigString(arena, config_string);
Assert((size_t)config.num_devices == expected.size());
for (int i = 0; i < config.num_devices; ++i) {
Assert(config.block_sizes[i] == expected[i]);
}
}

void TestBlockSizes(Arena *arena) {
std::string base_config = "num_devices = 4;\n";

hermes::ParseConfig(&arena, argv[1], &config);
{
std::vector<int> expected{50, 50, 50, 50};
std::string config_string = "block_sizes_bytes = {50, 50, 50, 50};\n";
RunBlockSizesTest(arena, base_config + config_string, expected);
}
{
std::vector<int> expected{KILOBYTES(50), KILOBYTES(50), KILOBYTES(50),
KILOBYTES(50)};
std::string config_string = "block_sizes_kb = {50, 50, 50, 50};\n";
RunBlockSizesTest(arena, base_config + config_string, expected);
}
{
std::vector<int> expected{MEGABYTES(50), MEGABYTES(50), MEGABYTES(50),
MEGABYTES(50)};
std::string config_string = "block_sizes_mb = {50, 50, 50, 50};\n";
RunBlockSizesTest(arena, base_config + config_string, expected);
}
{
std::vector<int> expected{GIGABYTES(1), GIGABYTES(1), GIGABYTES(1),
GIGABYTES(1)};
std::string config_string = "block_sizes_gb = {1, 1, 1, 1};\n";
RunBlockSizesTest(arena, base_config + config_string, expected);
}
}

void TestDefaultConfig(Arena *arena, const char *config_file) {
hermes::Config config = {};
hermes::ParseConfig(arena, config_file, &config);

Assert(config.num_devices == 4);
Assert(config.num_targets == 4);
Expand Down Expand Up @@ -153,10 +222,26 @@ int main(int argc, char **argv) {
Assert(config.is_shared_device[3] == 0);

Assert(config.bo_num_threads == 4);
}

Assert(config.default_rr_split == false);
} // namespace testing
} // namespace hermes

int main(int argc, char **argv) {
if (argc < 2) {
fprintf(stderr, "Expected a path to a hermes.conf file\n");
exit(-1);
}

const size_t kConfigMemorySize = KILOBYTES(16);
u8 config_memory[kConfigMemorySize];
Arena arena = {};
hermes::InitArena(&arena, kConfigMemorySize, config_memory);

TestParseRangeList(&arena);
hermes::testing::TestDefaultConfig(&arena, argv[1]);
hermes::testing::TestParseRangeList(&arena);
hermes::testing::TestCapacityValues(&arena);
hermes::testing::TestBlockSizes(&arena);

return 0;
}

0 comments on commit 69a0390

Please sign in to comment.