Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

nbd: add auto map at boot #347

Merged
merged 1 commit into from
May 19, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions curve-ansible/roles/install_package/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ local_nbd_package_path: ../../nbd-package
snapshot_clone_server_log_dir: /data/log/curve/snapshotclone
local_snapshotcloneserver_package_path: ../curve-snapshotcloneserver
local_monitor_package_path: ../../curve-monitor
curvetab_path: /etc/curve

local_snapshotcloneserver_nginx_package_path: ../curve-nginx
snapshotcloneserver_nginx_dir: /etc/curve/nginx
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,29 @@
local_file_path: "{{ local_nbd_package_path }}/bin/"
file_mode: 0755
include_tasks: copy_file_to_remote.yml

- name: install curvetab
vars:
remote_dir_name: "{{ curvetab_path }}"
local_file_path: "{{ local_nbd_package_path }}/etc/"
file_mode: 0755
include_tasks: copy_file_to_remote.yml

- name: add nbd auto start at boot
block:
- name: get distro name
vars:
distro:
include_tasks: common_tasks/get_distro_name.yml
- name: copy map_curve_disk.sh to init.d
vars:
remote_dir_name: "/etc/init.d"
local_file_path: "{{ curve_bin_dir }}/map_curve_disk.sh"
file_mode: 0755
include_tasks: copy_file_to_remote.yml
- name: enable on debain ubuntu
shell: sudo update-rc.d map_curve_disk.sh defaults
when: "'Ubuntu' in distro or 'Debian' in distro"
- name: enable on centos
shell: sudo chkconfig --add map_curve_disk.sh && sudo chkconfig map_curve_disk.sh on
when: "'CentOS' in distro"
3 changes: 3 additions & 0 deletions mk-tar.sh
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,10 @@ cp bazel-bin/nebd/src/part2/nebd-server build/nebd-package/bin

# step 4.2 prepare for curve-nbd package
mkdir -p build/nbd-package/bin
mkdir -p build/nbd-package/etc
cp bazel-bin/nbd/src/curve-nbd build/nbd-package/bin
cp nbd/nbd-package/usr/bin/map_curve_disk.sh build/nbd-package/bin
cp nbd/nbd-package/etc/curve/curvetab build/nbd-package/etc

#step5 打包tar包
echo "start make tarball"
Expand Down
9 changes: 9 additions & 0 deletions nbd/nbd-package/DEBIAN/postinst
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/bin/bash

set -e

chmod +x /usr/bin/map_curve_disk.sh
cp /usr/bin/map_curve_disk.sh /etc/init.d
update-rc.d map_curve_disk.sh defaults

exit 0
8 changes: 8 additions & 0 deletions nbd/nbd-package/DEBIAN/prerm
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash

set -e

update-rc.d map_curve_disk.sh remove
rm -f /etc/init.d/map_curve_disk.sh

exit 0
9 changes: 9 additions & 0 deletions nbd/nbd-package/etc/curve/curvetab
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#
# /etc/curvetab
# use for auto map and mount curve disk when boot
#
# this file record the correspondence of curve nbd images/ devices
# add record when 'curve-nbd map' and 'curve-nbd unmap'
#
# format: device image mount
# /dev/nbd0 cbd:pool//curvefile_test_ /curve
117 changes: 117 additions & 0 deletions nbd/nbd-package/usr/map_curve_disk.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
#!/bin/bash

### BEGIN INIT INFO
# Provides: curve-nbd
# Required-Start: $local_fs $remote_fs $network nebd-daemon
# Required-Stop: $local_fs $remote_fs $network
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: curve-nbd auto map service
# Description: curve-nbd auto map service and associated helpers
### END INIT INFO

# default confpath
# file format is:dealflag \t device \t image \t mountpoint(option)
# + /dev/nbd0 cbd:pool//curvefile_test_ /test
confPath=/etc/curve/curvetab

# check sudo
if [[ $(id -u) -ne 0 ]]
then
echo "Please run with sudo"
exit 1
fi

# usage
function usage() {
echo "Usage: ./map_curve_disk.sh start"
echo " -c/--confPath: set the confPath (default /etc/curve/curvetab)"
echo " file format is:dealflag \t device \t image \t mountpoint(option)"
echo " example: + /dev/nbd0 cbd:pool//curvefile_test_ /test"
echo " -h/--help: get the script usage"
echo "Examples:"
echo " ./map_curve_disk.sh start //use the default configuration"
echo " ./map_curve_disk.sh start --confPath yourConfPath //use the new confPath"
}

# remove extra records
function dealcurvetab() {
if [ ! -f ${confPath} ]
then
echo "ERROR: not found configuration file, confPath is ${confPath}!"
exit
fi

declare -A recordmap
while read line
do
flag=$(echo $line | awk '{print $1}')
device=$(echo $line | awk '{print $2}')
if [ "$flag" = "+" ]
then
recordmap["$device"]=$line
elif [ "$flag" = "-" ]
then
unset recordmap[$device]
fi
done < ${confPath}
for key in ${!recordmap[@]}; do
echo ${recordmap[$key]} >> ${confPath}.bak
done
}

# map the curve disk based on curvetab
function map() {
if [ ! -f ${confPath}.bak ]
then
echo "ERROR: not found configuration file, confPath is ${confPath}.bak!"
exit
fi

cat ${confPath}.bak | grep -v '#' | while read line
do
device=$(echo $line | awk '{print $2}')
image=$(echo $line | awk '{print $3}')
mount=$(echo $line | awk '{print $4}')
curve-nbd map $image --device $device
if [ "$mount" != "" ]
then
mount $device $mount
fi
done
mv ${confPath}.bak ${confPath}
}

if [ $# -lt 1 ]
then
usage
exit
fi

case $1 in
"start")
shift # pass first argument

while [[ $# -gt 1 ]]
do
key=$1

case $key in
-c|--confPath)
confPath=`realpath $2`
shift # pass key
shift # pass value
;;
*)
usage
exit
;;
esac
done
dealcurvetab
map
;;
*)
usage
;;
esac
1 change: 1 addition & 0 deletions nbd/src/define.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ namespace nbd {
#define PROCESS_NAME "curve-nbd"
#define NBD_PATH_PREFIX "/sys/block/nbd"
#define DEV_PATH_PREFIX "/dev/nbd"
#define CURVETAB_PATH "/etc/curve/curvetab"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe it's better to named CURVETAB_LOG_PATH ?


using std::cerr;

Expand Down
26 changes: 26 additions & 0 deletions nbd/src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,24 @@ static void Usage() {
<< std::endl;
}

// use for record image and device info to auto map when boot
static int AddRecord(int flag) {
std::string record;
int fd = open(CURVETAB_PATH, O_WRONLY | O_APPEND);
if (fd < 0) {
std::cerr << "curve-nbd: open curvetab file failed.";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

return

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

return

done

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

return with error code?

return -EINVAL;
}
if (1 == flag) {
record = "+\t" + nbdConfig->devpath + "\t" + nbdConfig->imgname + "\n";
} else if (-1 == flag) {
record = "-\t" + nbdConfig->devpath + "\n";
}
write(fd, record.c_str(), record.size());
close(fd);
return 0;
}

static int NBDConnect() {
int waitConnectPipe[2];

Expand Down Expand Up @@ -140,6 +158,10 @@ static int NBDConnect() {
if (ret < 0) {
::write(waitConnectPipe[1], &connectionRes, sizeof(connectionRes));
} else {
ret = AddRecord(1);
if (0 != ret) {
return ret;
}
connectionRes = 0;
::write(waitConnectPipe[1], &connectionRes, sizeof(connectionRes));
nbdTool->RunServerUntilQuit();
Expand Down Expand Up @@ -192,6 +214,10 @@ static int CurveNbdMain(int argc, const char* argv[]) {
return -EINVAL;
}

r = AddRecord(-1);
if (0 != r) {
return r;
}
break;
}
case Command::List: {
Expand Down