Faster remote publishing using rsync

 * Faster remote publishing using rsync over SSH: Previously the plug-in
   used Vim's netrw plug-in which started a new SSH connection for every
   file. Now a single connection can be used to upload all files if
   rsync is available both locally and on the remote side (the plug-in
   checks for this every time Publish() is called with a target path
   starting with sftp:// because rsync can give such a speed boost).
   While testing this change on my Vim profile* the total time to
   publish went down from 1 minute and 22 seconds to 54 seconds.

 * Replaced ad-hoc :echom code with xolox#message() and xolox#warning(),
   added dependency on ~/.vim/autoload/xolox.vim in Makefile

Makefile

@@ -1,5 +1,7 @@
-DEPENDS=$(HOME)/.vim/autoload/xolox/path.vim \
-		$(HOME)/.vim/autoload/xolox/escape.vim
+DEPENDS=autoload/xolox.vim \
+		autoload/xolox/escape.vim \
+		autoload/xolox/timer.vim \
+		autoload/xolox/path.vim
 VIMDOC=doc/publish.txt
 HTMLDOC=doc/readme.html
 ZIPDIR := $(shell mktemp -d)
@@ -14,7 +16,7 @@ archive: Makefile publish.vim autoload.vim $(VIMDOC) $(HTMLDOC)
 	@mkdir -p $(ZIPDIR)/plugin $(ZIPDIR)/autoload/xolox $(ZIPDIR)/doc
 	@cp publish.vim $(ZIPDIR)/plugin
 	@cp autoload.vim $(ZIPDIR)/autoload/publish.vim
-	@cp $(DEPENDS) $(ZIPDIR)/autoload/xolox
+	@for SCRIPT in $(DEPENDS); do cp $$HOME/.vim/$$SCRIPT $(ZIPDIR)/$$SCRIPT; done
 	@cp $(VIMDOC) $(ZIPDIR)/doc/publish.txt
 	@cp $(HTMLDOC) $(ZIPDIR)/doc/publish.html
 	@cd $(ZIPDIR) && zip -r $(ZIPFILE) . >/dev/null

README.md

@@ -50,6 +50,25 @@ script as `~/publish_test.vim` and execute it in Vim by typing `:source
 busy for a moment and after that you will find a bunch of syntax highlighted,
 interlinked HTML documents in the `target` directory!
 
+## Publishing to a remote location (website)
+
+As you can see from the example above it's possible to publish files directly
+to your web server using the [netrw.vim plug-in] [netrw] that's bundled with
+Vim, simply by starting the `target` path with `sftp://`. All you need for this
+to work is the ability to establish [SCP] [scp] connections to your server.
+There are however two disadvantages to remote publishing over [SFTP] [sftp]:
+
+1. The `publish.vim` plug-in can't automatically create directories on the
+remote side, which means you'll have to do so by hand -- very bothersome.
+
+2. It can take a while to publish a dozen files because a new connection is
+established for every file that's uploaded to the remote location.
+
+As a workaround to both of these issues the `publish.vim` plug-in will
+automatically use [rsync][rsync] when both the local and remote system have it
+installed. This cuts the time to publish to a remote location in half and
+enables the plug-in to automatically create directories on the remote side.
+
 ## Contact
 
 If you have questions, bug reports, suggestions, etc. the author can be
@@ -67,6 +86,11 @@ This software is licensed under the [MIT license] [license].<br>
 [ctags]: http://ctags.sourceforge.net/
 [demo]: http://peterodding.com/code/vim/profile/plugin/publish.vim
 [license]: http://en.wikipedia.org/wiki/MIT_License
+[netrw]: http://www.vim.org/scripts/script.php?script_id=1075
+[rsync]: http://en.wikipedia.org/wiki/rsync
+[scp]: http://en.wikipedia.org/wiki/Secure_copy
+[sftp]: http://en.wikipedia.org/wiki/SSH_file_transfer_protocol
+[ssh]: http://en.wikipedia.org/wiki/Secure_Shell
 [vim]: http://www.vim.org/
 [vim_scripts_entry]: http://www.vim.org/scripts/script.php?script_id=2252
 [zip]: http://peterodding.com/code/vim/download.php?script=publish

TODO.md

@@ -15,10 +15,6 @@ Here's some things in the nice-to-have department:
    rerun the plug-in when I break my [easytags.vim][easytags] plug-in which
    also breaks the hyperlinking feature of `publish.vim`.
 
- * Create an option to publish to a temporary local directory, create a tarball
-   from the published files, upload the tarball to a remote location and unpack
-   it there because establishing SFTP connections has quite a lot of overhead?
-
 
 [autoindex]: http://peterodding.com/code/vim/profile
 [easytags]: http://peterodding.com/code/vim/easytags

autoload.vim

@@ -39,19 +39,15 @@ function! publish#find_tags(files_to_publish) " {{{1
           let this_path = string(entry.filename)
           let other_path = string(other.filename)
           let msg = "publish.vim: Ignoring duplicate tag %s! (duplicate is in %s, first was in %s)"
-          echohl warningmsg
-          echomsg printf(msg, tag_name, this_path, other_path)
-          echohl none
+          call xolox#warning(msg, tag_name, this_path, other_path)
         endif
       endif
     endif
   endfor
   if num_duplicates > 3
     let more = num_duplicates - 3
     let msg = "publish.vim: Ignored %s more duplicate tag%s!"
-    echohl warningmsg
-    echomsg printf(msg, more, more == 1 ? '' : 's')
-    echohl none
+    call xolox#warning(msg, more, more == 1 ? '' : 's')
   endif
   unlet s:cached_contents
   return tags_to_publish
@@ -129,6 +125,32 @@ function! s:nasty()
   return '\%(' . short . '\|' . long . '\)'
 endfunction
 
+function! publish#rsync_check(target) " {{{1
+  let matches = matchlist(a:target, '^sftp://\([^/]\+\)\(.*\)$')
+  if len(matches) >= 3
+    let host = matches[1]
+    let path = substitute(matches[2], '^/', '', '')
+    call system('rsync --version')
+    if !v:shell_error
+      call system('ssh ' . host . ' rsync --version')
+      if !v:shell_error
+        return host . ':' . path
+      endif
+    endif
+  endif
+  return ''
+endfunction
+
+function! publish#run_rsync(target, tempdir) " {{{1
+  let target = fnameescape(a:target . '/')
+  let tempdir = fnameescape(a:tempdir . '/')
+  call xolox#message("Publishing files to %s using rsync..", a:target)
+  execute '!rsync -vr' tempdir target
+  if v:shell_error
+    throw "publish.vim: Failed to run rsync!"
+  endif
+endfunction
+
 function! publish#create_dirs(target_path) " {{{1
   " If the directory where the files are published resides on the local file
   " system then try to automatically create any missing directories because
@@ -141,11 +163,11 @@ function! publish#create_dirs(target_path) " {{{1
         let msg = "Failed to create directory %s! What now?"
         if confirm(printf(msg, string(current_directory)), "&Abort\n&Ignore") == 1
           let msg = "publish.vim: Failed to create %s, aborting .."
-          echomsg printf(msg, string(current_directory))
+          call xolox#warning(msg, string(current_directory))
           return 0
         else
           let msg = "publish.vim: Failed to create %s, ignoring .."
-          echomsg printf(msg, string(current_directory))
+          call xolox#warning(msg, string(current_directory))
           continue
         endif
       endif

publish.vim

@@ -1,9 +1,9 @@
 " Vim plug-in
 " Maintainer: Peter Odding <peter@peterodding.com>
-" Last Change: June 15, 2010
+" Last Change: June 16, 2010
 " URL: http://peterodding.com/code/vim/publish
 " License: MIT
-" Version: 1.5
+" Version: 1.6
 
 " Support for automatic update using the GLVS plug-in.
 " GetLatestVimScripts: 2252 1 :AutoInstall: publish.zip
@@ -26,18 +26,24 @@ if !exists('g:publish_viml_sl_hack')
 endif
 
 function! Publish(source, target, files) abort
-  call s:Message("Preparing to publish file%s ..", len(a:files) == 1 ? '' : 's')
+  let start = xolox#timer#start()
+  call xolox#message("Preparing to publish file%s ..", len(a:files) == 1 ? '' : 's')
   let s:files_to_publish = publish#resolve_files(a:source, a:files)
   let s:tags_to_publish = publish#find_tags(s:files_to_publish)
   if s:tags_to_publish != {}
     let tags_to_links_command = publish#create_subst_cmd(s:tags_to_publish)
   endif
+  let rsync_target = publish#rsync_check(a:target)
+  if rsync_target != ''
+    let rsync_dir = xolox#path#tempdir()
+  endif
+  let target_dir = rsync_target != '' ? rsync_dir : a:target
   call publish#prep_env(1)
   for pathname in a:files
     let source_path = xolox#path#merge(a:source, pathname)
     let suffix = g:publish_omit_dothtml ? '' : '.html'
-    let target_path = xolox#path#merge(a:target, pathname . suffix)
-    call s:Message("Publishing %s", string(pathname))
+    let target_path = xolox#path#merge(target_dir, pathname . suffix)
+    call xolox#message("Publishing %s", string(pathname))
     if !publish#create_dirs(target_path)
       return
     endif
@@ -47,7 +53,7 @@ function! Publish(source, target, files) abort
     let s:current_source_directory = fnamemodify(given_source, ':h')
     silent execute 'edit!' fnameescape(source_path)
     if g:publish_plaintext
-      let plaintext_path = xolox#path#merge(a:target, pathname . '.txt')
+      let plaintext_path = xolox#path#merge(target_dir, pathname . '.txt')
       silent execute 'write!' fnameescape(plaintext_path)
     endif
     silent execute 'doautocmd User PublishPre'
@@ -61,25 +67,19 @@ function! Publish(source, target, files) abort
   endfor
   call publish#prep_env(0)
   unlet s:files_to_publish s:tags_to_publish
-  let [msg, nfiles] = ["publish.vim: Published %i file%s to %s.", len(a:files)]
-  call s:Message(msg, nfiles, nfiles == 1 ? '' : 's', string(a:target))
+  if rsync_target != ''
+    call publish#run_rsync(rsync_target, rsync_dir)
+  endif
+  let msg = "publish.vim: Published %i file%s to %s."
+  call xolox#message(msg, len(a:files), len(a:files) == 1 ? '' : 's', a:target)
+  call xolox#timer#stop("Finished publishing files in %s.", start)
 endfunction
 
 function! s:FindOriginalPath(pathname) " {{{1
   let key = xolox#path#absolute(a:pathname)
   return get(s:files_to_publish, key, '')
 endfunction
 
-function! s:Message(...) " {{{1
-  try
-    redraw
-    echohl title
-    echomsg call('printf', a:000)
-  finally
-    echohl none
-  endtry
-endfunction
-
 function! s:ConvertTagToLink(name) " {{{1
   " Convert each occurrence of every tag into a hyperlink that points to the
   " location where the tag is defined. Since the hyperlinks are relative they
@@ -96,6 +96,7 @@ function! s:ConvertTagToLink(name) " {{{1
     " Convert the fully resolved pathname back into the one given by the user.
     let pathname = s:FindOriginalPath(entry.filename)
     " Now convert that pathname into a relative hyperlink with an anchor.
+    " TODO This is likely to be slow so cache the results?!
     let relative = xolox#path#relative(pathname, s:current_source_directory)
     let suffix = g:publish_omit_dothtml ? '' : '.html'
     return '<a href="' . relative . suffix . '#l' . entry.lnum . '">' . a:name . '</a>'