Add FTP support to file_splitter.rb

Allows for sending both anonymous and authorized FTP uploads directly
from the script.  Splits are done in memory, so the file never is
created on the host machine, which is ideal for large backups and such
that are being split up.

Spec for testing this functionality are done via live FTP (vsftpd)
server setup with the following config:

    listen=NO
    listen_ipv6=YES

    local_enable=YES
    local_umask=022
    write_enable=YES
    connect_from_port_20=YES

    anonymous_enable=YES
    anon_root=/var/ftp/pub
    anon_umask=022
    anon_upload_enable=YES
    anon_mkdir_write_enable=YES
    anon_other_write_enable=YES

    pam_service_name=vsftpd
    userlist_enable=YES
    userlist_deny=NO
    tcp_wrappers=YES

The specs will only run if the user specifically targets the tests using
a tag flag:

    $ bundle exec rspec --tag with_real_ftp

lib/manageiq/util/file_splitter.rb

@@ -13,10 +13,11 @@
 # platform, without having to be concerned with differences in `split`
 # functionality.
 
+require 'net/ftp'
 require 'optparse'
 
 class FileSplitter
-  attr_accessor :input_file, :byte_count
+  attr_accessor :input_file, :byte_count, :ftp
 
   def self.run(options = nil)
     options ||= parse_argv
@@ -29,6 +30,12 @@ def self.parse_argv
       opt.on("-b", "--byte-count=BYTES", "Number of bytes for each split") do |bytes|
         options[:byte_count] = parse_byte_value(bytes)
       end
+      opt.on("--ftp-host=HOST", "Host of the FTP server") do |host|
+        options[:ftp_host] = host
+      end
+      opt.on("--ftp-dir=DIR",   "Dir on the FTP server to save files") do |dir|
+        options[:ftp_dir] = dir
+      end
     end.parse!
 
     input_file, file_pattern = determine_input_file_and_file_pattern
@@ -44,21 +51,84 @@ def initialize(options = {})
     @input_filename = options[:input_filename]
     @byte_count     = options[:byte_count] || 10_485_760
     @position       = 0
+
+    setup_ftp(options)
   end
 
   def split
     until input_file.eof?
-      File.open(next_split_filename, "w") do |split_file|
-        split_file << input_file.read(byte_count)
-        @position += byte_count
+      if ftp
+        split_ftp
+      else
+        split_local
       end
+      @position += byte_count
     end
   ensure
     input_file.close
+    ftp.close if ftp
   end
 
   private
 
+  def setup_ftp(options)
+    if options[:ftp_host]
+      @ftp = ::Net::FTP.new(options[:ftp_host])
+      user = options[:ftp_user] || ENV["FTP_USERNAME"] || "anonymous"
+      pass = options[:ftp_pass] || ENV["FTP_PASSWORD"]
+      @ftp.login(user, pass)
+
+      @input_filename = File.join(options[:ftp_dir] || "", File.basename(input_filename))
+    end
+  end
+
+  def split_local
+    File.open(next_split_filename, "w") do |split_file|
+      split_file << input_file.read(byte_count)
+    end
+  end
+
+  # Specific version of Net::FTP#storbinary that doesn't use an existing local
+  # file, and only uploads a specific size from the input_file
+  FTP_CHUNKSIZE = ::Net::FTP::DEFAULT_BLOCKSIZE
+  def split_ftp
+    ftp_mkdir_p
+    ftp.synchronize do
+      ftp.send(:with_binary, true) do
+        conn         = ftp.send(:transfercmd, "STOR #{next_split_filename}")
+        buf_left     = byte_count
+        while buf_left > 0 do
+          cur_readsize = buf_left - FTP_CHUNKSIZE >= 0 ? FTP_CHUNKSIZE : buf_left
+          buf = input_file.read(cur_readsize)
+          break if buf == nil
+          conn.write(buf)
+          buf_left -= FTP_CHUNKSIZE
+        end
+        conn.close
+        ftp.send(:voidresp)
+      end
+    end
+  rescue Errno::EPIPE
+    # EPIPE, in this case, means that the data connection was unexpectedly
+    # terminated.  Rather than just raising EPIPE to the caller, check the
+    # response on the control connection.  If getresp doesn't raise a more
+    # appropriate exception, re-raise the original exception.
+    getresp
+    raise
+  end
+
+  # Taken from FileDepotFtp#create_directory_structure
+  def ftp_mkdir_p
+    pwd = ftp.pwd
+    (File.dirname(input_filename)[1..-1].split('/') - pwd[1..-1].split("/")).each do |directory|
+      unless ftp.nlst.include?(directory)
+        ftp.mkdir(directory)
+      end
+      ftp.chdir(directory)
+    end
+    ftp.chdir(pwd)
+  end
+
   def input_filename
     @input_filename ||= File.expand_path(input_file.path)
   end

spec/lib/manageiq/util/file_splitter_spec.rb

@@ -2,6 +2,18 @@
 require 'pathname'
 require 'manageiq/util/file_splitter'
 
+# Putting this here since this config option can die with this script, and
+# doesn't need to live in the global config.
+RSpec.configure do |config|
+  # These are tests that shouldn't run on CI, but should be semi-automated to
+  # be triggered manaually to test in an automated fasion.  There will be setup
+  # steps with a vagrant file to spinup a endpoint to use for this.
+  #
+  # TODO:  Maybe we should just use VCR for this?  Still would required the
+  # vagrant VM I guess to record the tests from... so for now, skipping.
+  config.filter_run_excluding :with_real_ftp => true
+end
+
 describe "file_splitter.rb" do
   shared_context "generated tmp files" do
     let!(:tmpfile_size) { 10.megabytes }
@@ -21,6 +33,25 @@
     end
   end
 
+  shared_context "ftp context" do
+    let(:ftp) do
+      Net::FTP.new(ftp_host).tap do |ftp|
+        ftp.login(ftp_creds[:ftp_user], ftp_creds[:ftp_pass])
+      end
+    end
+
+    let(:ftp_dir)    { File.join("", "uploads") }
+    let(:ftp_host)   { ENV["FTP_HOST_FOR_SPECS"] || "192.168.50.3" }
+    let(:ftp_creds)  { { :ftp_user => "anonymous", :ftp_pass => nil } }
+    let(:ftp_config) { { :ftp_host => ftp_host, :ftp_dir => ftp_dir } }
+
+    let(:ftp_user_1) { ENV["FTP_USER_1_FOR_SPECS"] || "vagrant" }
+    let(:ftp_pass_1) { ENV["FTP_USER_1_FOR_SPECS"] || "vagrant" }
+
+    let(:ftp_user_2) { ENV["FTP_USER_2_FOR_SPECS"] || "foo" }
+    let(:ftp_pass_2) { ENV["FTP_USER_2_FOR_SPECS"] || "bar" }
+  end
+
   describe ".run" do
     include_context "generated tmp files"
 
@@ -58,6 +89,112 @@
         expect(Pathname.new(filename).lstat.size).to eq(1.megabyte)
       end
     end
+
+    context "to an ftp target", :with_real_ftp => true do
+      include_context "ftp context"
+
+      let(:base_config) { { :byte_count => 1.megabyte, :input_file => File.open(source_path) } }
+      let(:run_config)  { ftp_config.merge(base_config) }
+
+      let(:expected_splitfiles) do
+        (1..10).map do |suffix|
+          File.join(ftp_dir, "#{source_path.basename}.000#{'%02d' % suffix}")
+        end
+      end
+
+      it "uploads the split files as an annoymous user by default" do
+        FileSplitter.run(run_config)
+
+        expected_splitfiles.each do |filename|
+          expect(ftp.nlst(filename)).to eq([filename])
+          expect(ftp.size(filename)).to eq(1.megabyte)
+          ftp.delete(filename)
+        end
+      end
+
+      context "with slightly a slightly smaller input file than 10MB" do
+        let(:tmpfile_size) { 10.megabytes - 1.kilobyte }
+
+        it "properly chunks the file" do
+          FileSplitter.run(run_config)
+
+          expected_splitfiles[0,9].each do |filename|
+            expect(ftp.nlst(filename)).to eq([filename])
+            expect(ftp.size(filename)).to eq(1.megabyte)
+            ftp.delete(filename)
+          end
+
+          expect(ftp.nlst(expected_splitfiles.last)).to eq([expected_splitfiles.last])
+          expect(ftp.size(expected_splitfiles.last)).to eq(1.megabyte - 1.kilobyte)
+          ftp.delete(expected_splitfiles.last)
+        end
+      end
+
+      context "with slightly a slightly larger input file than 10MB" do
+        let(:tmpfile_size) { 10.megabytes + 1.kilobyte }
+
+        it "properly chunks the file" do
+          FileSplitter.run(run_config)
+
+          expected_splitfiles.each do |filename|
+            expect(ftp.nlst(filename)).to eq([filename])
+            expect(ftp.size(filename)).to eq(1.megabyte)
+            ftp.delete(filename)
+          end
+
+          filename = File.join(ftp_dir, "#{source_path.basename}.00011")
+          expect(ftp.nlst(filename)).to eq([filename])
+          expect(ftp.size(filename)).to eq(1.kilobyte)
+          ftp.delete(filename)
+        end
+      end
+
+      context "with a dir that doesn't exist" do
+        let(:ftp_dir) { File.join("", "uploads", "backups", "current") }
+
+        it "uploads the split files" do
+          FileSplitter.run(run_config)
+
+          expected_splitfiles.each do |filename|
+            expect(ftp.nlst(filename)).to eq([filename])
+            expect(ftp.size(filename)).to eq(1.megabyte)
+            ftp.delete(filename)
+          end
+          ftp.rmdir(ftp_dir)
+        end
+      end
+
+      context "with a specified user" do
+        let(:ftp_dir)    { File.join("", "home", ftp_user_1) }
+        let(:ftp_creds)  { { :ftp_user => ftp_user_1, :ftp_pass => ftp_pass_1 } }
+        let(:run_config) { ftp_config.merge(ftp_creds).merge(base_config) }
+
+        it "uploads the split files" do
+          FileSplitter.run(run_config)
+
+          expected_splitfiles.each do |filename|
+            expect(ftp.nlst(filename)).to eq([filename])
+            expect(ftp.size(filename)).to eq(1.megabyte)
+            ftp.delete(filename)
+          end
+        end
+
+        context "with a dir that doesn't exist" do
+          let(:ftp_dir) { File.join("", "home", ftp_user_1, "backups") }
+
+          it "uploads the split files" do
+            FileSplitter.run(run_config)
+
+            expected_splitfiles.each do |filename|
+              expect(ftp.nlst(filename)).to eq([filename])
+              expect(ftp.size(filename)).to eq(1.megabyte)
+              ftp.delete(filename)
+            end
+            ftp.rmdir(ftp_dir)
+          end
+        end
+      end
+    end
   end
 
   describe ".parse_argv" do
@@ -174,5 +311,48 @@
         expect(Pathname.new(filename).lstat.size).to eq(2.megabyte)
       end
     end
+
+    context "to a ftp target", :with_real_ftp => true do
+      include_context "ftp context"
+
+      let(:expected_splitfiles) do
+        (1..5).map do |suffix|
+          File.join(ftp_dir, "#{source_path.basename}.000#{'%02d' % suffix}")
+        end
+      end
+
+      it "it uploads with an anonymous user by default" do
+        cmd_opts = "-b 2M --ftp-host=#{ftp_host} --ftp-dir #{ftp_dir}"
+        `cat #{source_path.expand_path} | #{script_file} #{cmd_opts} - #{source_path.basename}`
+
+        expected_splitfiles.each do |filename|
+          expect(ftp.nlst(filename)).to eq([filename])
+          expect(ftp.size(filename)).to eq(2.megabyte)
+          ftp.delete(filename)
+        end
+      end
+
+      context "with a specified user via ENV vars" do
+        let(:ftp_dir)    { File.join("", "home", ftp_user_1, "backups") }
+        let(:ftp_creds)  { { :ftp_user => ftp_user_1, :ftp_pass => ftp_pass_1 } }
+        let(:run_config) { ftp_config.merge(ftp_creds).merge(base_config) }
+
+        it "uploads the split files and creates necessary dirs" do
+          env      = { 'FTP_USERNAME' => ftp_user_1, 'FTP_PASSWORD' => ftp_pass_1 }
+          cmd_opts = "-b 2M --ftp-host=#{ftp_host} --ftp-dir #{ftp_dir}"
+          cmd      = "cat #{source_path.expand_path} | #{script_file} #{cmd_opts} - #{source_path.basename}"
+          pid      = Kernel.spawn(env, cmd)
+          Process.wait(pid)
+
+          expect($?).to eq(0)
+          expected_splitfiles.each do |filename|
+            expect(ftp.nlst(filename)).to eq([filename])
+            expect(ftp.size(filename)).to eq(2.megabyte)
+            ftp.delete(filename)
+          end
+          ftp.rmdir(ftp_dir)
+        end
+      end
+    end
   end
 end