Skip to content

Commit

Permalink
nbd: fix concurrent nbd map
Browse files Browse the repository at this point in the history
  • Loading branch information
@wu-hanqing
wu-hanqing committed Mar 25, 2021
1 parent 501ada9 commit 0c5f459
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 93 deletions.
56 changes: 19 additions & 37 deletions nbd/src/NBDController.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,9 @@
namespace curve {
namespace nbd {

int IOController::InitDevAttr(int devfd, NBDConfig* config, int sockfd,
int IOController::InitDevAttr(int devfd, NBDConfig* config,
uint64_t size, uint64_t flags) {
int ret = ioctl(devfd, NBD_SET_SOCK, sockfd);
if (ret < 0) {
dout << "curve-ndb: the device " << config->devpath
<< " is busy" << std::endl;
return -errno;
}
int ret = -1;

do {
ret = ioctl(devfd, NBD_SET_BLKSIZE, CURVE_NBD_BLKSIZE);
Expand All @@ -74,52 +69,39 @@ int IOController::InitDevAttr(int devfd, NBDConfig* config, int sockfd,
if (config->timeout >= 0) {
ret = ioctl(devfd, NBD_SET_TIMEOUT, (unsigned long)config->timeout); // NOLINT
if (ret < 0) {
dout << "curve-ndb: failed to set timeout: "
dout << "curve-nbd: failed to set timeout: "
<< cpp_strerror(ret) << std::endl;
break;
}
}
} while (false);

if (ret < 0) {
ret = -errno;
ioctl(devfd, NBD_CLEAR_SOCK);
}
return ret;
}

int IOController::SetUp(NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags) {
if (config->devpath.empty()) {
config->devpath = find_unused_nbd_device();
}
int devfd = -1;
int index = -1;

if (config->devpath.empty()) {
dout << "devpath is empty" << std::endl;
return -1;
}

int ret = parse_nbd_index(config->devpath);
if (ret < 0) {
return ret;
devfd = map_on_unused_nbd_device(sockfd, &config->devpath, &index);
} else {
devfd = map_on_nbd_device_by_devpath(sockfd, config->devpath, &index);
}
int index = ret;

ret = open(config->devpath.c_str(), O_RDWR);
if (ret < 0) {
dout << "curve-ndb: failed to open device: "
<< config->devpath << std::endl;
return ret;
if (devfd == -1) {
return -1;
}
int devfd = ret;

ret = InitDevAttr(devfd, config, sockfd, size, flags);
int ret = InitDevAttr(devfd, config, size, flags);
if (ret == 0) {
ret = check_device_size(index, size);
}
if (ret < 0) {
dout << "curve-ndb: failed to map, status: "
dout << "curve-nbd: failed to map, status: "
<< cpp_strerror(ret) << std::endl;
ioctl(devfd, NBD_CLEAR_SOCK);
close(devfd);
return ret;
}
Expand All @@ -132,14 +114,14 @@ int IOController::SetUp(NBDConfig* config, int sockfd,
int IOController::DisconnectByPath(const std::string& devpath) {
int devfd = open(devpath.c_str(), O_RDWR);
if (devfd < 0) {
dout << "curve-ndb: failed to open device: "
dout << "curve-nbd: failed to open device: "
<< devpath << ", error = " << cpp_strerror(errno) << std::endl;
return devfd;
}

int ret = ioctl(devfd, NBD_DISCONNECT);
if (ret < 0) {
dout << "curve-ndb: the device is not used. "
dout << "curve-nbd: the device is not used. "
<< cpp_strerror(errno) << std::endl;
}

Expand Down Expand Up @@ -397,7 +379,7 @@ int NetLinkController::DisconnectInternal(int index) {
genl_handle_msg, NULL);
msg = nlmsg_alloc();
if (msg == nullptr) {
dout << "curve-ndb: Could not allocate netlink message." << std::endl;
dout << "curve-nbd: Could not allocate netlink message." << std::endl;
return -EIO;
}

Expand All @@ -412,7 +394,7 @@ int NetLinkController::DisconnectInternal(int index) {

ret = nl_send_sync(sock_, msg);
if (ret < 0) {
dout << "curve-ndb: netlink disconnect failed: "
dout << "curve-nbd: netlink disconnect failed: "
<< nl_geterror(ret) << std::endl;
return -EIO;
}
Expand All @@ -432,7 +414,7 @@ int NetLinkController::ResizeInternal(int nbdIndex, uint64_t size) {
genl_handle_msg, NULL);
msg = nlmsg_alloc();
if (msg == nullptr) {
dout << "curve-ndb: Could not allocate netlink message." << std::endl;
dout << "curve-nbd: Could not allocate netlink message." << std::endl;
return -EIO;
}

Expand All @@ -448,7 +430,7 @@ int NetLinkController::ResizeInternal(int nbdIndex, uint64_t size) {

ret = nl_send_sync(sock_, msg);
if (ret < 0) {
dout << "curve-ndb: netlink resize failed: "
dout << "curve-nbd: netlink resize failed: "
<< nl_geterror(ret) << std::endl;
return -EIO;
}
Expand Down
4 changes: 2 additions & 2 deletions nbd/src/NBDController.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ class IOController : public NBDController {
int Resize(uint64_t size) override;

private:
int InitDevAttr(int devfd, NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags);
int InitDevAttr(int devfd, NBDConfig* config, uint64_t size,
uint64_t flags);
};

class NetLinkController : public NBDController {
Expand Down
116 changes: 68 additions & 48 deletions nbd/src/util.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,54 +89,6 @@ int get_nbd_max_count() {
return nbds_max;
}

std::string find_unused_nbd_device() {
int index = 0;
int devfd = 0;
int nbds_max = get_nbd_max_count();
char dev[64];
int sockfd[2];

int ret = socketpair(AF_UNIX, SOCK_STREAM, 0, sockfd);
if (ret < 0) {
dout << "curve-ndb: failed to create socket pair." << std::endl;
return "";
}

while (true) {
snprintf(dev, sizeof(dev), "/dev/nbd%d", index);

ret = open(dev, O_RDWR);
if (ret < 0) {
if (ret == -EPERM && nbds_max != -1 && index < (nbds_max-1)) {
++index;
continue;
}
dout << "curve-ndb: failed to find unused device, "
<< cpp_strerror(errno) << std::endl;
break;
}

devfd = ret;
ret = ioctl(devfd, NBD_SET_SOCK, sockfd[0]);
if (ret < 0) {
close(devfd);
++index;
continue;
}
break;
}

std::string result = "";
if (ret == 0) {
result = dev;
ioctl(devfd, NBD_CLEAR_SOCK);
close(devfd);
}
close(sockfd[0]);
close(sockfd[1]);
return result;
}

static bool find_mapped_dev_by_spec(NBDConfig *cfg) {
int pid;
NBDConfig c;
Expand Down Expand Up @@ -441,6 +393,74 @@ int load_module(NBDConfig *cfg) {
return ret;
}

int map_on_unused_nbd_device(int sockfd, std::string* devpath, int* nbdindex) {
int index = 0;
int devfd = -1;
char dev[64];
const int nbds_max = get_nbd_max_count();

while (index < nbds_max) {
// open device
snprintf(dev, sizeof(dev), "/dev/nbd%d", index);
devfd = open(dev, O_RDWR);
if (devfd < 0) {
if (devfd == -EPERM) {
++index;
continue;
} else {
dout << "curve-nbd: failed to map on unused device, "
<< cpp_strerror(errno) << std::endl;
break;
}
}

// set sock
int ret = ioctl(devfd, NBD_SET_SOCK, sockfd);
if (ret < 0) {
close(devfd);
devfd = -1;
++index;
continue;
} else {
break;
}
}

if (devfd < 0) {
dout << "curve-nbd: faild to map on unused device, nbds_max: "
<< nbds_max << ", last try nbd index: " << index << std::endl;
return -1;
} else {
*devpath = dev;
*nbdindex = index;
return devfd;
}
}

int map_on_nbd_device_by_devpath(int sockfd, const std::string& devpath,
int* nbdindex) {
int index = parse_nbd_index(devpath);
if (index < 0) {
return -1;
}

int devfd = open(devpath.c_str(), O_RDWR);
if (devfd < 0) {
dout << "curve-nbd: failed to open device: " << devpath << std::endl;
return -1;
}

int ret = ioctl(devfd, NBD_SET_SOCK, sockfd);
if (ret < 0) {
dout << "curve-nbd: ioctl NBD_SET_SOCK failed, devpath = " << devpath
<< ", error = " << cpp_strerror(errno) << std::endl;
return -1;
}

*nbdindex = index;
return devfd;
}

bool NBDListIterator::Get(int *pid, NBDConfig *cfg) {
while (true) {
std::string nbd_path = NBD_PATH_PREFIX + std::to_string(curIndex_);
Expand Down
6 changes: 4 additions & 2 deletions nbd/src/util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ extern std::string cpp_strerror(int err);
extern int parse_nbd_index(const std::string& devpath);
// 获取当前系统能够支持的最大nbd设备数量
extern int get_nbd_max_count();
// 获取一个当前还未映射的nbd设备名
extern std::string find_unused_nbd_device();
// 解析用户输入的命令参数
extern int parse_args(std::vector<const char*>& args, // NOLINT
std::ostream *err_msg,
Expand All @@ -63,6 +61,10 @@ extern int check_device_size(int nbd_index, uint64_t expected_size);
// 如果当前系统还未加载nbd模块，则进行加载；如果已经加载，则不作任何操作
extern int load_module(NBDConfig *cfg);

int map_on_unused_nbd_device(int sockfd, std::string* devpath, int* nbdindex);
int map_on_nbd_device_by_devpath(int sockfd, const std::string& devpath,
int* nbdindex);

// 安全读写文件或socket，对异常情况进行处理后返回
ssize_t safe_read_exact(int fd, void* buf, size_t count);
ssize_t safe_read(int fd, void* buf, size_t count);
Expand Down
13 changes: 9 additions & 4 deletions test/common/test_timeutility.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,25 @@ TEST(ExpiredTimeTest, CommonTest) {
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::seconds(2));
auto expiredSec = expiredTime.ExpiredSec();
ASSERT_TRUE(expiredSec >= 1.8 && expiredSec <= 2.2);
double expected = 2;
ASSERT_GE(expiredSec, expected * 0.9);
ASSERT_LE(expiredSec, expected * 1.1);
}
{
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
auto expiredMs = expiredTime.ExpiredMs();
ASSERT_TRUE(expiredMs >= (1000 - 10) && expiredMs <= (1000 + 10));
double expected = 1000;
ASSERT_GE(expiredMs, expected * 0.9);
ASSERT_LE(expiredMs, expected * 1.1);
}
{
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::microseconds(1000000));
auto expiredUs = expiredTime.ExpiredUs();
ASSERT_TRUE(expiredUs >= (1000000 - 200) &&
expiredUs <= (1000000 + 200));
double expected = 1000000;
ASSERT_GE(expiredUs, expected * 0.9);
ASSERT_LE(expiredUs, expected * 1.1);
}
}

Expand Down

0 comments on commit 0c5f459

Please sign in to comment.