Skip to content

Commit

Permalink
Merge pull request #11830 from adamemerson/wip-crush-unlocked
Browse files Browse the repository at this point in the history 
crush: remove the crush_lock

Reviewed-by: Sage Weil <sage@redhat.com>
  • Loading branch information
@liewegas
liewegas committed Nov 11, 2016
2 parents 26a3dae + cafdc65 commit a1303ce
Show file tree
Hide file tree
Showing 11 changed files with 236 additions and 188 deletions.
3 changes: 0 additions & 3 deletions src/crush/CrushWrapper.cc
View file
Expand Up @@ -1365,9 +1365,6 @@ void CrushWrapper::decode_crush_bucket(crush_bucket** bptr, bufferlist::iterator
::decode(bucket->items[j], blp);
}

bucket->perm = (__u32*)calloc(1, bucket->size * sizeof(__u32));
bucket->perm_n = 0;

switch (bucket->alg) {
case CRUSH_BUCKET_UNIFORM:
::decode((reinterpret_cast<crush_bucket_uniform*>(bucket))->item_weight, blp);
Expand Down
20 changes: 9 additions & 11 deletions src/crush/CrushWrapper.h
View file
Expand Up @@ -50,7 +50,6 @@ inline static void decode(crush_rule_step &s, bufferlist::iterator &p)

using namespace std;
class CrushWrapper {
mutable Mutex mapper_lock;
public:
std::map<int32_t, string> type_map; /* bucket/device type names */
std::map<int32_t, string> name_map; /* bucket/device names */
Expand Down Expand Up @@ -78,8 +77,7 @@ class CrushWrapper {
CrushWrapper(const CrushWrapper& other);
const CrushWrapper& operator=(const CrushWrapper& other);

CrushWrapper() : mapper_lock("CrushWrapper::mapper_lock"),
crush(0), have_rmaps(false) {
CrushWrapper() : crush(0), have_rmaps(false) {
create();
}
~CrushWrapper() {
Expand Down Expand Up @@ -1088,26 +1086,26 @@ class CrushWrapper {

void do_rule(int rule, int x, vector<int>& out, int maxout,
const vector<__u32>& weight) const {
Mutex::Locker l(mapper_lock);
int rawout[maxout];
int scratch[maxout * 3];
int numrep = crush_do_rule(crush, rule, x, rawout, maxout, &weight[0], weight.size(), scratch);
char work[crush_work_size(crush, maxout)];
crush_init_workspace(crush, work);
int numrep = crush_do_rule(crush, rule, x, rawout, maxout, &weight[0],
weight.size(), work);
if (numrep < 0)
numrep = 0;
out.resize(numrep);
for (int i=0; i<numrep; i++)
out[i] = rawout[i];
}

bool check_crush_rule(int ruleset, int type, int size, ostream& ss) {

assert(crush);
assert(crush);

__u32 i;
for (i = 0; i < crush->max_rules; i++) {
if (crush->rules[i] &&
crush->rules[i]->mask.ruleset == ruleset &&
crush->rules[i]->mask.type == type) {
crush->rules[i]->mask.ruleset == ruleset &&
crush->rules[i]->mask.type == type) {

if (crush->rules[i]->mask.min_size <= size &&
crush->rules[i]->mask.max_size >= size) {
Expand Down
101 changes: 19 additions & 82 deletions src/crush/builder.c
View file
Expand Up @@ -45,6 +45,13 @@ void crush_finalize(struct crush_map *map)
int b;
__u32 i;

/* Calculate the needed working space while we do other
finalization tasks. */
map->working_size = sizeof(struct crush_work);
/* Space for the array of pointers to per-bucket workspace */
map->working_size += map->max_buckets *
sizeof(struct crush_work_bucket *);

/* calc max_devices */
map->max_devices = 0;
for (b=0; b<map->max_buckets; b++) {
Expand All @@ -53,13 +60,21 @@ void crush_finalize(struct crush_map *map)
for (i=0; i<map->buckets[b]->size; i++)
if (map->buckets[b]->items[i] >= map->max_devices)
map->max_devices = map->buckets[b]->items[i] + 1;

switch (map->buckets[b]->alg) {
default:
/* The base case, permutation variables and
the pointer to the permutation array. */
map->working_size += sizeof(struct crush_work_bucket);
break;
}
/* Every bucket has a permutation array. */
map->working_size += map->buckets[b]->size * sizeof(__u32);
}
}





/** rules **/

int crush_add_rule(struct crush_map *map, struct crush_rule *rule, int ruleno)
Expand Down Expand Up @@ -212,16 +227,11 @@ crush_make_uniform_bucket(int hash, int type, int size,
if (!bucket->h.items)
goto err;

bucket->h.perm = malloc(sizeof(__u32)*size);

if (!bucket->h.perm)
goto err;
for (i=0; i<size; i++)
bucket->h.items[i] = items[i];

return bucket;
err:
free(bucket->h.perm);
free(bucket->h.items);
free(bucket);
return NULL;
Expand Down Expand Up @@ -251,9 +261,6 @@ crush_make_list_bucket(int hash, int type, int size,
bucket->h.items = malloc(sizeof(__s32)*size);
if (!bucket->h.items)
goto err;
bucket->h.perm = malloc(sizeof(__u32)*size);
if (!bucket->h.perm)
goto err;


bucket->item_weights = malloc(sizeof(__u32)*size);
Expand Down Expand Up @@ -282,7 +289,6 @@ crush_make_list_bucket(int hash, int type, int size,
err:
free(bucket->sum_weights);
free(bucket->item_weights);
free(bucket->h.perm);
free(bucket->h.items);
free(bucket);
return NULL;
Expand Down Expand Up @@ -347,7 +353,6 @@ crush_make_tree_bucket(int hash, int type, int size,

if (size == 0) {
bucket->h.items = NULL;
bucket->h.perm = NULL;
bucket->h.weight = 0;
bucket->node_weights = NULL;
bucket->num_nodes = 0;
Expand All @@ -358,9 +363,6 @@ crush_make_tree_bucket(int hash, int type, int size,
bucket->h.items = malloc(sizeof(__s32)*size);
if (!bucket->h.items)
goto err;
bucket->h.perm = malloc(sizeof(__u32)*size);
if (!bucket->h.perm)
goto err;

/* calc tree depth */
depth = calc_depth(size);
Expand Down Expand Up @@ -399,7 +401,6 @@ crush_make_tree_bucket(int hash, int type, int size,
return bucket;
err:
free(bucket->node_weights);
free(bucket->h.perm);
free(bucket->h.items);
free(bucket);
return NULL;
Expand Down Expand Up @@ -577,9 +578,6 @@ crush_make_straw_bucket(struct crush_map *map,
bucket->h.items = malloc(sizeof(__s32)*size);
if (!bucket->h.items)
goto err;
bucket->h.perm = malloc(sizeof(__u32)*size);
if (!bucket->h.perm)
goto err;
bucket->item_weights = malloc(sizeof(__u32)*size);
if (!bucket->item_weights)
goto err;
Expand All @@ -601,7 +599,6 @@ crush_make_straw_bucket(struct crush_map *map,
err:
free(bucket->straws);
free(bucket->item_weights);
free(bucket->h.perm);
free(bucket->h.items);
free(bucket);
return NULL;
Expand Down Expand Up @@ -630,9 +627,6 @@ crush_make_straw2_bucket(struct crush_map *map,
bucket->h.items = malloc(sizeof(__s32)*size);
if (!bucket->h.items)
goto err;
bucket->h.perm = malloc(sizeof(__u32)*size);
if (!bucket->h.perm)
goto err;
bucket->item_weights = malloc(sizeof(__u32)*size);
if (!bucket->item_weights)
goto err;
Expand All @@ -647,7 +641,6 @@ crush_make_straw2_bucket(struct crush_map *map,
return bucket;
err:
free(bucket->item_weights);
free(bucket->h.perm);
free(bucket->h.items);
free(bucket);
return NULL;
Expand Down Expand Up @@ -698,11 +691,6 @@ int crush_add_uniform_bucket_item(struct crush_bucket_uniform *bucket, int item,
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}

bucket->h.items[newsize-1] = item;

Expand All @@ -725,11 +713,6 @@ int crush_add_list_bucket_item(struct crush_bucket_list *bucket, int item, int w
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}
if ((_realloc = realloc(bucket->item_weights, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
Expand Down Expand Up @@ -775,17 +758,12 @@ int crush_add_tree_bucket_item(struct crush_bucket_tree *bucket, int item, int w
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}
if ((_realloc = realloc(bucket->node_weights, sizeof(__u32)*bucket->num_nodes)) == NULL) {
return -ENOMEM;
} else {
bucket->node_weights = _realloc;
}

node = crush_calc_tree_node(newsize-1);
bucket->node_weights[node] = weight;

Expand Down Expand Up @@ -824,19 +802,14 @@ int crush_add_straw_bucket_item(struct crush_map *map,
int item, int weight)
{
int newsize = bucket->h.size + 1;

void *_realloc = NULL;

if ((_realloc = realloc(bucket->h.items, sizeof(__s32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}
if ((_realloc = realloc(bucket->item_weights, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
Expand Down Expand Up @@ -873,11 +846,6 @@ int crush_add_straw2_bucket_item(struct crush_map *map,
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}
if ((_realloc = realloc(bucket->item_weights, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
Expand All @@ -899,9 +867,6 @@ int crush_add_straw2_bucket_item(struct crush_map *map,
int crush_bucket_add_item(struct crush_map *map,
struct crush_bucket *b, int item, int weight)
{
/* invalidate perm cache */
b->perm_n = 0;

switch (b->alg) {
case CRUSH_BUCKET_UNIFORM:
return crush_add_uniform_bucket_item((struct crush_bucket_uniform *)b, item, weight);
Expand Down Expand Up @@ -945,11 +910,6 @@ int crush_remove_uniform_bucket_item(struct crush_bucket_uniform *bucket, int it
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}
return 0;
}

Expand Down Expand Up @@ -984,11 +944,6 @@ int crush_remove_list_bucket_item(struct crush_bucket_list *bucket, int item)
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}
if ((_realloc = realloc(bucket->item_weights, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
Expand Down Expand Up @@ -1053,11 +1008,6 @@ int crush_remove_tree_bucket_item(struct crush_bucket_tree *bucket, int item)
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}

olddepth = calc_depth(bucket->h.size);
newdepth = calc_depth(newsize);
Expand Down Expand Up @@ -1106,11 +1056,6 @@ int crush_remove_straw_bucket_item(struct crush_map *map,
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}
if ((_realloc = realloc(bucket->item_weights, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
Expand Down Expand Up @@ -1155,11 +1100,6 @@ int crush_remove_straw2_bucket_item(struct crush_map *map,
} else {
bucket->h.items = _realloc;
}
if ((_realloc = realloc(bucket->h.perm, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
bucket->h.perm = _realloc;
}
if ((_realloc = realloc(bucket->item_weights, sizeof(__u32)*newsize)) == NULL) {
return -ENOMEM;
} else {
Expand All @@ -1171,9 +1111,6 @@ int crush_remove_straw2_bucket_item(struct crush_map *map,

int crush_bucket_remove_item(struct crush_map *map, struct crush_bucket *b, int item)
{
/* invalidate perm cache */
b->perm_n = 0;

switch (b->alg) {
case CRUSH_BUCKET_UNIFORM:
return crush_remove_uniform_bucket_item((struct crush_bucket_uniform *)b, item);
Expand Down
5 changes: 0 additions & 5 deletions src/crush/crush.c
View file
Expand Up @@ -45,7 +45,6 @@ int crush_get_bucket_item_weight(const struct crush_bucket *b, int p)

void crush_destroy_bucket_uniform(struct crush_bucket_uniform *b)
{
kfree(b->h.perm);
kfree(b->h.items);
kfree(b);
}
Expand All @@ -54,14 +53,12 @@ void crush_destroy_bucket_list(struct crush_bucket_list *b)
{
kfree(b->item_weights);
kfree(b->sum_weights);
kfree(b->h.perm);
kfree(b->h.items);
kfree(b);
}

void crush_destroy_bucket_tree(struct crush_bucket_tree *b)
{
kfree(b->h.perm);
kfree(b->h.items);
kfree(b->node_weights);
kfree(b);
Expand All @@ -71,15 +68,13 @@ void crush_destroy_bucket_straw(struct crush_bucket_straw *b)
{
kfree(b->straws);
kfree(b->item_weights);
kfree(b->h.perm);
kfree(b->h.items);
kfree(b);
}

void crush_destroy_bucket_straw2(struct crush_bucket_straw2 *b)
{
kfree(b->item_weights);
kfree(b->h.perm);
kfree(b->h.items);
kfree(b);
}
Expand Down

0 comments on commit a1303ce

Please sign in to comment.