Use real mount points

Previously, we would create one loopback device with some subdirs
to represent "drives" and symlink them into place under /srv/node*

Now, use N loopback devices. Note that you may want to crank down
LOOPBACK_GB as it applies to each sparse file and the default may
allow you to fill your VM's root disk.

Two side benefits (beyond getting closer to "real" system behavior):
we can now enable mount_check, and we can have per-node-and-service
rsync modules.

Addresses #43.

bin/cleanswift

@@ -19,11 +19,13 @@ set -e
 swift-init all stop || :
 find /var/log/swift -type f -delete || :
 find /var/cache/swift* -type f -name *.recon -delete
-if mount | grep -q swift-disk ; then
-    sudo umount /mnt/swift-disk
-fi
+
+for d in $(ls -d /srv/node*/*); do
+    mount | grep -q $d && sudo umount $d
+done
 sudo rm -rf /srv/node*
-sudo rm -f /var/lib/swift/disk
+sudo rm -f /var/lib/swift/disk*
+
 for fname in /var/log/debug /var/log/messages /var/log/rsyncd.log /var/log/syslog; do
     sudo truncate --size 0 $fname
 done

cookbooks/swift/files/default/etc/swift/account-server/default.conf-template

@@ -1,5 +1,4 @@
 [DEFAULT]
-mount_check = false
 disable_fallocate = true
 workers = 1
 

cookbooks/swift/files/default/etc/swift/container-server/default.conf-template

@@ -1,5 +1,4 @@
 [DEFAULT]
-mount_check = false
 disable_fallocate = true
 workers = 1
 

cookbooks/swift/recipes/data.rb

@@ -25,37 +25,6 @@
   end
 end
 
-execute "create sparse file" do
-  command "truncate -s #{node['loopback_gb']}GB /var/lib/swift/disk"
-  creates "/var/lib/swift/disk"
-  action :run
-end
-
-execute "create file system" do
-  command "mkfs.xfs /var/lib/swift/disk"
-  not_if "xfs_admin -l /var/lib/swift/disk"
-  action :run
-end
-
-execute "update fstab" do
-  command "echo '/var/lib/swift/disk /mnt/swift-disk xfs " \
-    "loop,noatime,nodiratime,nobarrier,logbufs=8 0 0' >> /etc/fstab"
-  not_if "grep swift-disk /etc/fstab"
-  action :run
-end
-
-execute "mount" do
-  command "mount /mnt/swift-disk"
-  not_if "mountpoint /mnt/swift-disk"
-end
-
-# for unittest xfs scratch
-directory "/mnt/swift-disk/tmp" do
-  owner node["username"]
-  group node["username"]
-  action :create
-end
-
 if node['ec_policy'].empty? then
   num_disks = node['disks']
 else
@@ -64,25 +33,58 @@
 
 (1..num_disks).each do |i|
   j = ((i - 1) % node['nodes']) + 1
-  disk_path = "/mnt/swift-disk/sdb#{i}"
+  disk_file = "/var/lib/swift/disk#{i}"
   node_path = "/srv/node#{j}"
-  srv_path = node_path + "/sdb#{i}"
-  directory disk_path do
-    owner node["username"]
-    group node["username"]
+  mount_path = node_path + "/sdb#{i}"
+
+
+  execute "create sparse file #{i}" do
+    command "truncate -s #{node['loopback_gb']}GB #{disk_file}"
+    creates "#{disk_file}"
+    action :run
+  end
+
+  execute "create file system" do
+    command "mkfs.xfs #{disk_file}"
+    not_if "xfs_admin -l #{disk_file}"
+    action :run
+  end
+
+  directory mount_path do
+    owner 'root'
+    group 'root'
+    mode '0755'
+    recursive true
     action :create
   end
-  directory "create node path for #{disk_path}" do
-    path node_path
+
+  execute "update fstab" do
+    command "echo '#{disk_file} #{mount_path} xfs " \
+      "loop,noatime,nodiratime,nobarrier,logbufs=8 0 0' >> /etc/fstab"
+    not_if "grep #{mount_path} /etc/fstab"
+    action :run
+  end
+
+  execute "mount" do
+    command "mount #{mount_path}"
+    not_if "mountpoint #{mount_path}"
+  end
+
+  # Fix perms on mounted dir
+  directory mount_path do
     owner node["username"]
     group node["username"]
-    action :create
-  end
-  link srv_path do
-    to disk_path 
+    mode '0775'
   end
 end
 
+# for unittest xfs scratch
+directory "/srv/node1/sdb1/tmp" do
+  owner node["username"]
+  group node["username"]
+  action :create
+end
+
 # run dirs
 
 [

cookbooks/swift/recipes/setup.rb

@@ -158,7 +158,7 @@
 {
   "SOURCE_ROOT" => "#{node['source_root']}",
   "NOSE_INCLUDE_EXE" => "true",
-  "TMPDIR" => "/mnt/swift-disk/tmp",
+  "TMPDIR" => "/srv/node1/sdb1/tmp",
 }.each do |var, value|
   execute "swift-env-#{var}" do
     command "echo 'export #{var}=#{value}' >> #{profile_file}"

cookbooks/swift/templates/default/etc/swift/object-server/default.conf-template.erb

@@ -1,5 +1,4 @@
 [DEFAULT]
-mount_check = false
 disable_fallocate = true
 workers = 1
 log_facility = LOG_LOCAL2