nbd: add auto map at boot

curve-ansible/roles/install_package/defaults/main.yml

@@ -30,6 +30,7 @@ local_nbd_package_path: ../../nbd-package
 snapshot_clone_server_log_dir: /data/log/curve/snapshotclone
 local_snapshotcloneserver_package_path: ../curve-snapshotcloneserver
 local_monitor_package_path: ../../curve-monitor
+curvetab_path: /etc/curve
 
 local_snapshotcloneserver_nginx_package_path: ../curve-nginx
 snapshotcloneserver_nginx_dir: /etc/curve/nginx

curve-ansible/roles/install_package/tasks/include/install_curve-nbd.yml

@@ -45,3 +45,29 @@
     local_file_path: "{{ local_nbd_package_path }}/bin/"
     file_mode: 0755
   include_tasks: copy_file_to_remote.yml
+
+- name: install curvetab
+  vars:
+    remote_dir_name: "{{ curvetab_path }}"
+    local_file_path: "{{ local_nbd_package_path }}/etc/"
+    file_mode: 0755
+  include_tasks: copy_file_to_remote.yml
+
+- name: add nbd auto start at boot
+  block:
+    - name: get distro name
+      vars:
+        distro:
+      include_tasks: common_tasks/get_distro_name.yml
+    - name: copy map_curve_disk.sh to init.d
+      vars:
+        remote_dir_name: "/etc/init.d"
+        local_file_path: "{{ curve_bin_dir }}/map_curve_disk.sh"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: enable on debain ubuntu
+      shell: sudo update-rc.d map_curve_disk.sh defaults
+      when: "'Ubuntu' in distro or 'Debian' in distro"
+    - name: enable on centos
+      shell: sudo chkconfig --add map_curve_disk.sh && sudo chkconfig map_curve_disk.sh on
+      when: "'CentOS' in distro"

mk-tar.sh

@@ -461,7 +461,10 @@ cp bazel-bin/nebd/src/part2/nebd-server build/nebd-package/bin
 
 # step 4.2 prepare for curve-nbd package
 mkdir -p build/nbd-package/bin
+mkdir -p build/nbd-package/etc
 cp bazel-bin/nbd/src/curve-nbd build/nbd-package/bin
+cp nbd/nbd-package/usr/bin/map_curve_disk.sh build/nbd-package/bin
+cp nbd/nbd-package/etc/curve/curvetab build/nbd-package/etc
 
 #step5 打包tar包
 echo "start make tarball"

nbd/nbd-package/DEBIAN/postinst

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+chmod +x /usr/bin/map_curve_disk.sh
+cp /usr/bin/map_curve_disk.sh /etc/init.d
+update-rc.d map_curve_disk.sh defaults
+
+exit 0

nbd/nbd-package/DEBIAN/prerm

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+update-rc.d map_curve_disk.sh remove
+rm -f /etc/init.d/map_curve_disk.sh
+
+exit 0

nbd/nbd-package/etc/curve/curvetab

@@ -0,0 +1,9 @@
+#
+# /etc/curvetab
+# use for auto map and mount curve disk when boot
+#
+# this file record the correspondence of curve nbd images/ devices
+# add record when 'curve-nbd map' and delete when 'curve-nbd unmap'
+#
+# format: device    image   mount
+# cbd:pool//curvefile_test_   /dev/nbd0   /curve

nbd/src/define.h

@@ -61,6 +61,7 @@ namespace nbd {
 #define PROCESS_NAME "curve-nbd"
 #define NBD_PATH_PREFIX "/sys/block/nbd"
 #define DEV_PATH_PREFIX "/dev/nbd"
+#define CURVETAB_PATH "/etc/curve/curvetab"
 
 using std::cerr;
 

nbd/src/main.cpp

@@ -23,6 +23,8 @@
  */
 
 #include <sys/wait.h>
+#include <sys/file.h>
+#include <ext/stdio_filebuf.h>
 #include <unistd.h>
 
 #include <iostream>
@@ -95,6 +97,112 @@ static void Usage() {
         << std::endl;
 }
 
+// use for record image and device info to auto map when boot
+static void AddRecord() {
+    std::fstream fs;
+    int fd;
+    fs.open(CURVETAB_PATH, std::fstream::in);
+    if (!fs.is_open()) {
+        std::cerr << "curve-nbd: open curvetab file failed.";
+        return;
+    }
+    fd = static_cast<__gnu_cxx::stdio_filebuf< char > * const >
+                                            (fs.rdbuf())->fd();
+    if (0 == flock(fd, LOCK_SH )) {
+        // check if the record exists
+        std::string lineString;
+        while (getline(fs, lineString)) {
+            if (!lineString.empty() && lineString[0] != '#') {
+                std::vector<std::string> lineArray;
+                std::stringstream ss(lineString);
+                std::string str;
+                while (getline(ss, str, '\t')) {
+                    lineArray.push_back(str);
+                }
+                if (lineArray.size() != 2 && lineArray.size() != 3) {
+                    std::cerr << "curve-nbd: curvetab file content error.";
+                    fs.close();
+                    return;
+                }
+                if (lineArray[0] == nbdConfig->imgname) {
+                    fs.close();
+                    return;
+                }
+            }
+        }
+        fs.close();
+        flock(fd, LOCK_UN);
+    }
+    // write new record
+    fs.open(CURVETAB_PATH, std::fstream::app);
+    if (!fs.is_open()) {
+        std::cerr << "curve-nbd: open curvetab file failed.";
+        return;
+    }
+    fd = static_cast<__gnu_cxx::stdio_filebuf< char > * const>
+                                           (fs.rdbuf())->fd();
+    if (0 == flock(fd, LOCK_EX)) {
+        fs << nbdConfig->imgname << '\t' << nbdConfig->devpath << '\n';
+        fs.close();
+        flock(fd, LOCK_UN);
+    }
+}
+
+// delete record when unmap
+static void DelRecord() {
+    std::fstream fs;
+    int fd;
+    fs.open(CURVETAB_PATH, std::fstream::in);
+    if (!fs.is_open()) {
+        std::cerr << "curve-nbd: open curvetab file failed.";
+        return;
+    }
+    fd = static_cast<__gnu_cxx::stdio_filebuf< char > * const>
+                                           (fs.rdbuf())->fd();
+    std::string lineString;
+    std::string newContent;
+    if (0 == flock(fd, LOCK_SH)) {
+        // find record need deleted
+        while (getline(fs, lineString)) {
+            if (!lineString.empty() && lineString[0] != '#') {
+                std::vector<std::string> lineArray;
+                std::stringstream ss(lineString);
+                std::string str;
+                while (getline(ss, str, '\t')) {
+                    lineArray.push_back(str);
+                }
+                if (lineArray.size() != 2 && lineArray.size() != 3) {
+                    std::cerr << "curve-nbd: curvetab file content error.";
+                    fs.close();
+                    return;
+                }
+                if (lineArray[1] != nbdConfig->devpath) {
+                    newContent += lineString;
+                    newContent += "\n";
+                }
+            } else {
+                newContent += lineString;
+                newContent += "\n";
+            }
+        }
+        fs.close();
+        flock(fd, LOCK_UN);
+    }
+    // update record info
+    fs.open(CURVETAB_PATH, std::fstream::out);
+    if (!fs.is_open()) {
+        std::cerr << "curve-nbd: open curvetab file failed.";
+        return;
+    }
+    fd = static_cast<__gnu_cxx::stdio_filebuf< char > * const>
+                                           (fs.rdbuf())->fd();
+    if (0 == flock(fd, LOCK_EX)) {
+        fs << newContent;
+        fs.close();
+        flock(fd, LOCK_UN);
+    }
+}
+
 static int NBDConnect() {
     int waitConnectPipe[2];
 
@@ -140,6 +248,7 @@ static int NBDConnect() {
     if (ret < 0) {
         ::write(waitConnectPipe[1], &connectionRes, sizeof(connectionRes));
     } else {
+        AddRecord();
         connectionRes = 0;
         ::write(waitConnectPipe[1], &connectionRes, sizeof(connectionRes));
         nbdTool->RunServerUntilQuit();
@@ -192,6 +301,7 @@ static int CurveNbdMain(int argc, const char* argv[]) {
                 return -EINVAL;
             }
 
+            DelRecord();
             break;
         }
         case Command::List: {