Skip to content

Commit

Permalink
nbd: fix concurrent nbd map
Browse files Browse the repository at this point in the history
  • Loading branch information
@wu-hanqing
wu-hanqing committed Mar 29, 2021
1 parent ff8d9da commit 2e9f178
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 100 deletions.
126 changes: 82 additions & 44 deletions nbd/src/NBDController.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,104 +42,142 @@
namespace curve {
namespace nbd {

int IOController::InitDevAttr(int devfd, NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags) {
int ret = ioctl(devfd, NBD_SET_SOCK, sockfd);
if (ret < 0) {
dout << "curve-ndb: the device " << config->devpath
<< " is busy" << std::endl;
return -errno;
}
int IOController::InitDevAttr(NBDConfig* config, uint64_t size,
uint64_t flags) {
int ret = -1;

do {
ret = ioctl(devfd, NBD_SET_BLKSIZE, CURVE_NBD_BLKSIZE);
ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, CURVE_NBD_BLKSIZE);
if (ret < 0) {
break;
}

ret = ioctl(devfd, NBD_SET_SIZE, size);
ret = ioctl(nbdFd_, NBD_SET_SIZE, size);
if (ret < 0) {
break;
}

ioctl(devfd, NBD_SET_FLAGS, flags);
ioctl(nbdFd_, NBD_SET_FLAGS, flags);

ret = CheckSetReadOnly(devfd, flags);
ret = CheckSetReadOnly(nbdFd_, flags);
if (ret < 0) {
dout << "curve-nbd: Check and set read only flag failed."
<< cpp_strerror(ret) << std::endl;
break;
}

if (config->timeout >= 0) {
ret = ioctl(devfd, NBD_SET_TIMEOUT, (unsigned long)config->timeout); // NOLINT
ret = ioctl(nbdFd_, NBD_SET_TIMEOUT, (unsigned long)config->timeout); // NOLINT
if (ret < 0) {
dout << "curve-ndb: failed to set timeout: "
dout << "curve-nbd: failed to set timeout: "
<< cpp_strerror(ret) << std::endl;
break;
}
}
} while (false);

if (ret < 0) {
ret = -errno;
ioctl(devfd, NBD_CLEAR_SOCK);
}
return ret;
}

int IOController::SetUp(NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags) {
if (config->devpath.empty()) {
config->devpath = find_unused_nbd_device();
int IOController::MapOnUnusedNbdDevice(int sockfd, std::string* devpath) {
int index = 0;
char dev[64];
const int nbdsMax = get_nbd_max_count();

while (index < nbdsMax) {
snprintf(dev, sizeof(dev), "/dev/nbd%d", index);

int ret = MapOnNbdDeviceByDevPath(sockfd, dev, false);
if (ret < 0) {
++index;
continue;
} else {
*devpath = dev;
return 0;
}
}

if (config->devpath.empty()) {
dout << "devpath is empty" << std::endl;
dout << "curve-nbd: failed to map on unused device, max nbd index: "
<< (nbdsMax - 1) << ", last try nbd index: " << (index - 1)
<< ", last error: " << cpp_strerror(errno) << std::endl;

return -1;
}

int IOController::MapOnNbdDeviceByDevPath(int sockfd,
const std::string& devpath,
bool logWhenError) {
int index = parse_nbd_index(devpath);
if (index < 0) {
return -1;
}

int devfd = open(devpath.c_str(), O_RDWR);
if (devfd < 0) {
if (logWhenError) {
dout << "curve-nbd: failed to open device: " << devfd
<< ", error = " << cpp_strerror(errno) << std::endl;
}
return -1;
}

int ret = parse_nbd_index(config->devpath);
int ret = ioctl(devfd, NBD_SET_SOCK, sockfd);
if (ret < 0) {
return ret;
if (logWhenError) {
dout << "curve-nbd: ioctl NBD_SET_SOCK failed, devpath: " << devpath
<< ", error = " << cpp_strerror(errno) << std::endl;
}
close(devfd);
return -1;
}

nbdFd_ = devfd;
nbdIndex_ = index;
return 0;
}

int IOController::SetUp(NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags) {
int ret = -1;

if (config->devpath.empty()) {
ret = MapOnUnusedNbdDevice(sockfd, &config->devpath);
} else {
ret = MapOnNbdDeviceByDevPath(sockfd, config->devpath);
}
int index = ret;

ret = open(config->devpath.c_str(), O_RDWR);
if (ret < 0) {
dout << "curve-ndb: failed to open device: "
<< config->devpath << std::endl;
return ret;
return -1;
}
int devfd = ret;

ret = InitDevAttr(devfd, config, sockfd, size, flags);
ret = InitDevAttr(config, size, flags);
if (ret == 0) {
ret = check_device_size(index, size);
ret = check_device_size(nbdIndex_, size);
}
if (ret < 0) {
dout << "curve-ndb: failed to map, status: "
dout << "curve-nbd: failed to map, status: "
<< cpp_strerror(ret) << std::endl;
close(devfd);
ioctl(nbdFd_, NBD_CLEAR_SOCK);
close(nbdFd_);
nbdFd_ = -1;
nbdIndex_ = -1;
return ret;
}

nbdFd_ = devfd;
nbdIndex_ = index;
return 0;
}

int IOController::DisconnectByPath(const std::string& devpath) {
int devfd = open(devpath.c_str(), O_RDWR);
if (devfd < 0) {
dout << "curve-ndb: failed to open device: "
dout << "curve-nbd: failed to open device: "
<< devpath << ", error = " << cpp_strerror(errno) << std::endl;
return devfd;
}

int ret = ioctl(devfd, NBD_DISCONNECT);
if (ret < 0) {
dout << "curve-ndb: the device is not used. "
dout << "curve-nbd: the device is not used. "
<< cpp_strerror(errno) << std::endl;
}

Expand Down Expand Up @@ -397,7 +435,7 @@ int NetLinkController::DisconnectInternal(int index) {
genl_handle_msg, NULL);
msg = nlmsg_alloc();
if (msg == nullptr) {
dout << "curve-ndb: Could not allocate netlink message." << std::endl;
dout << "curve-nbd: Could not allocate netlink message." << std::endl;
return -EIO;
}

Expand All @@ -412,7 +450,7 @@ int NetLinkController::DisconnectInternal(int index) {

ret = nl_send_sync(sock_, msg);
if (ret < 0) {
dout << "curve-ndb: netlink disconnect failed: "
dout << "curve-nbd: netlink disconnect failed: "
<< nl_geterror(ret) << std::endl;
return -EIO;
}
Expand All @@ -432,7 +470,7 @@ int NetLinkController::ResizeInternal(int nbdIndex, uint64_t size) {
genl_handle_msg, NULL);
msg = nlmsg_alloc();
if (msg == nullptr) {
dout << "curve-ndb: Could not allocate netlink message." << std::endl;
dout << "curve-nbd: Could not allocate netlink message." << std::endl;
return -EIO;
}

Expand All @@ -448,7 +486,7 @@ int NetLinkController::ResizeInternal(int nbdIndex, uint64_t size) {

ret = nl_send_sync(sock_, msg);
if (ret < 0) {
dout << "curve-ndb: netlink resize failed: "
dout << "curve-nbd: netlink resize failed: "
<< nl_geterror(ret) << std::endl;
return -EIO;
}
Expand Down
6 changes: 4 additions & 2 deletions nbd/src/NBDController.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,10 @@ class IOController : public NBDController {
int Resize(uint64_t size) override;

private:
int InitDevAttr(int devfd, NBDConfig* config, int sockfd,
uint64_t size, uint64_t flags);
int InitDevAttr(NBDConfig* config, uint64_t size, uint64_t flags);
int MapOnUnusedNbdDevice(int sockfd, std::string* devpath);
int MapOnNbdDeviceByDevPath(int sockfd, const std::string& devpath,
bool logWhenError = true);
};

class NetLinkController : public NBDController {
Expand Down
48 changes: 0 additions & 48 deletions nbd/src/util.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,54 +89,6 @@ int get_nbd_max_count() {
return nbds_max;
}

std::string find_unused_nbd_device() {
int index = 0;
int devfd = 0;
int nbds_max = get_nbd_max_count();
char dev[64];
int sockfd[2];

int ret = socketpair(AF_UNIX, SOCK_STREAM, 0, sockfd);
if (ret < 0) {
dout << "curve-ndb: failed to create socket pair." << std::endl;
return "";
}

while (true) {
snprintf(dev, sizeof(dev), "/dev/nbd%d", index);

ret = open(dev, O_RDWR);
if (ret < 0) {
if (ret == -EPERM && nbds_max != -1 && index < (nbds_max-1)) {
++index;
continue;
}
dout << "curve-ndb: failed to find unused device, "
<< cpp_strerror(errno) << std::endl;
break;
}

devfd = ret;
ret = ioctl(devfd, NBD_SET_SOCK, sockfd[0]);
if (ret < 0) {
close(devfd);
++index;
continue;
}
break;
}

std::string result = "";
if (ret == 0) {
result = dev;
ioctl(devfd, NBD_CLEAR_SOCK);
close(devfd);
}
close(sockfd[0]);
close(sockfd[1]);
return result;
}

static bool find_mapped_dev_by_spec(NBDConfig *cfg) {
int pid;
NBDConfig c;
Expand Down
2 changes: 0 additions & 2 deletions nbd/src/util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ extern std::string cpp_strerror(int err);
extern int parse_nbd_index(const std::string& devpath);
// 获取当前系统能够支持的最大nbd设备数量
extern int get_nbd_max_count();
// 获取一个当前还未映射的nbd设备名
extern std::string find_unused_nbd_device();
// 解析用户输入的命令参数
extern int parse_args(std::vector<const char*>& args, // NOLINT
std::ostream *err_msg,
Expand Down
13 changes: 9 additions & 4 deletions test/common/test_timeutility.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,25 @@ TEST(ExpiredTimeTest, CommonTest) {
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::seconds(2));
auto expiredSec = expiredTime.ExpiredSec();
ASSERT_TRUE(expiredSec >= 1.8 && expiredSec <= 2.2);
double expected = 2;
ASSERT_GE(expiredSec, expected * 0.9);
ASSERT_LE(expiredSec, expected * 1.1);
}
{
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
auto expiredMs = expiredTime.ExpiredMs();
ASSERT_TRUE(expiredMs >= (1000 - 10) && expiredMs <= (1000 + 10));
double expected = 1000;
ASSERT_GE(expiredMs, expected * 0.9);
ASSERT_LE(expiredMs, expected * 1.1);
}
{
ExpiredTime expiredTime;
std::this_thread::sleep_for(std::chrono::microseconds(1000000));
auto expiredUs = expiredTime.ExpiredUs();
ASSERT_TRUE(expiredUs >= (1000000 - 200) &&
expiredUs <= (1000000 + 200));
double expected = 1000000;
ASSERT_GE(expiredUs, expected * 0.9);
ASSERT_LE(expiredUs, expected * 1.1);
}
}

Expand Down

0 comments on commit 2e9f178

Please sign in to comment.