Use HTTP HEAD to disambiguate URLs with trailing punctuation (issue #4)

Author: xolox

README.md

@@ -83,6 +83,12 @@ Since no mappings will be defined now you can add something like the following t
     :inoremap <Leader>op <C-o>:Open<CR>
     :nnoremap <Leader>op :Open<CR>
 
+### The `g:shell_verify_urls` option
+
+When you use the `:Open` command or the `<F6>` mapping to open the URL under the text cursor, the shell plug-in uses a regular expression to guess where the URL starts and ends. This works 99% percent of the time but it can break, because in this process the shell plug-in will strip trailing punctuation characters like dots (because they were likely not intended to be included in the URL).
+
+If you actually deal with URLs that include significant trailing punctuation and your Vim is compiled with Python support you can enable the option `g:shell_verify_urls` (by setting it to 1 in your [vimrc script] [vimrc]). When you do this the plug-in will perform an HTTP HEAD request on the URL without stripping trailing punctuation. If the request returns an HTTP status code that indicates some form of success (the status code is at least 200 and less than 400) the URL including trailing punctuation is opened. If the HEAD request fails the plug-in will try again without trailing punctuation.
+
 ## Background
 
 Vim has a limited ability to call external libraries using the Vim script function [libcall()][libcall]. A few years ago when I was still using Windows a lot I created a [Windows DLL][dll] that could be used with [libcall()][libcall] to toggle [Vim][vim]'s GUI window between regular and full-screen mode. I also added a few other useful functions, e.g. `openurl()` to launch the default web browser and `execute()` which works like Vim's [system()][system] function but doesn't wait for the process to finish and doesn't show a command prompt.

autoload/xolox/shell.vim

@@ -1,9 +1,9 @@
 " Vim auto-load script
 " Author: Peter Odding <peter@peterodding.com>
-" Last Change: October 18, 2011
+" Last Change: October 28, 2011
 " URL: http://peterodding.com/code/vim/shell/
 
-let g:xolox#shell#version = '0.9.17'
+let g:xolox#shell#version = '0.9.18'
 
 if !exists('s:fullscreen_enabled')
   let s:enoimpl = "%s() hasn't been implemented on your platform! %s"
@@ -14,17 +14,22 @@ endif
 
 function! xolox#shell#open_cmd(arg) " -- implementation of the :Open command {{{1
   try
+    " No argument?
     if a:arg !~ '\S'
+      " Filename, URL or e-mail address at text cursor location?
       if !s:open_at_cursor()
+        " Open the directory of the current buffer.
         let bufdir = expand('%:p:h')
         call xolox#misc#msg#debug("shell.vim %s: Opening directory of current buffer '%s'.", g:xolox#shell#version, bufdir)
         call xolox#misc#open#file(bufdir)
       endif
     elseif (a:arg =~ xolox#shell#url_pattern()) || (a:arg =~ xolox#shell#mail_pattern())
+      " Open the URL or e-mail address given as an argument.
       call xolox#misc#msg#debug("shell.vim %s: Opening URL or e-mail address '%s'.", g:xolox#shell#version, a:arg)
       call xolox#misc#open#url(a:arg)
     else
       let arg = fnamemodify(a:arg, ':p')
+      " Does the argument point to an existing file or directory?
       if isdirectory(arg) || filereadable(arg)
         call xolox#misc#msg#debug("shell.vim %s: Opening valid filename '%s'.", g:xolox#shell#version, arg)
         call xolox#misc#open#file(arg)
@@ -42,7 +47,11 @@ function! s:open_at_cursor()
   let cWORD = expand('<cWORD>')
   " Start by trying to match a URL in <cWORD> because URLs can be more-or-less
   " unambiguously distinguished from e-mail addresses and filenames.
-  let match = matchstr(cWORD, xolox#shell#url_pattern())
+  if g:shell_verify_urls && cWORD =~ '^\(http\|https\)://.\{-}[[:punct:]]$' && xolox#shell#url_exists(cWORD)
+    let match = cWORD
+  else
+    let match = matchstr(cWORD, xolox#shell#url_pattern())
+  endif
   if match != ''
     call xolox#misc#msg#debug("shell.vim %s: Matched URL '%s' in cWORD '%s'.", g:xolox#shell#version, match, cWORD)
   else
@@ -248,6 +257,54 @@ function! xolox#shell#is_fullscreen() " -- check whether Vim is currently in ful
   return s:fullscreen_enabled
 endfunction
 
+function! xolox#shell#url_exists(url) " -- check whether a URL points to an existing resource (using Python) {{{1
+  try
+    " Embedding Python code in Vim scripts is always a bit awkward :-(
+    " (because of the forced indentation thing Python insists on).
+    let starttime = xolox#misc#timer#start()
+python <<EOF
+
+# Standard library modules.
+import httplib
+import urlparse
+
+# Only loaded inside the Python interface to Vim.
+import vim
+
+# We need to define a function to enable redirection implemented through recursion.
+
+def shell_url_exists(absolute_url, rec=0):
+  assert rec <= 10
+  components = urlparse.urlparse(absolute_url)
+  netloc = components.netloc.split(':', 1)
+  if components.scheme == 'http':
+    connection = httplib.HTTPConnection(*netloc)
+  elif components.scheme == 'https':
+    connection = httplib.HTTPSConnection(*netloc)
+  else:
+    assert False, "Unsupported URL scheme"
+  relative_url = urlparse.urlunparse(('', '') + components[2:])
+  connection.request('HEAD', relative_url)
+  response = connection.getresponse()
+  if 300 <= response.status < 400:
+    for name, value in response.getheaders():
+      if name.lower() == 'location':
+        shell_url_exists(value.strip(), rec+1)
+        break
+  else:
+    assert 200 <= response.status < 400
+
+shell_url_exists(vim.eval('a:url'))
+
+EOF
+    call xolox#misc#timer#stop("shell.vim %s: Took %s to verify whether %s exists (it does).", g:xolox#shell#version, starttime, a:url)
+    return 1
+  catch
+    call xolox#misc#timer#stop("shell.vim %s: Took %s to verify whether %s exists (it doesn't).", g:xolox#shell#version, starttime, a:url)
+    return 0
+  endtry
+endfunction
+
 function! xolox#shell#url_pattern() " -- get the preferred/default pattern to match URLs {{{1
   return xolox#misc#option#get('shell_patt_url', '\<\w\{3,}://\(\S*\w\)\+[/?#]\?')
 endfunction

doc/shell.txt

@@ -155,6 +155,25 @@ to your |vimrc| script:
     :inoremap <Leader>op <C-o>:Open<CR>
     :nnoremap <Leader>op :Open<CR>
 
+-------------------------------------------------------------------------------
+The *g:shell_verify_urls* option
+
+When you use the |:Open| command or the '<F6>' mapping to open the URL under
+the text cursor, the shell plug-in uses a regular expression to guess where
+the URL starts and ends. This works 99% percent of the time but it can break,
+because in this process the shell plug-in will strip trailing punctuation
+characters like dots (because they were likely not intended to be included in
+the URL).
+
+If you actually deal with URLs that include significant trailing punctuation
+and your Vim is compiled with Python support you can enable the option
+|g:shell_verify_urls| (by setting it to 1 in your |vimrc| script). When you do
+this the plug-in will perform an HTTP HEAD request on the URL without
+stripping trailing punctuation. If the request returns an HTTP status code
+that indicates some form of success (the status code is at least 200 and less
+than 400) the URL including trailing punctuation is opened. If the HEAD
+request fails the plug-in will try again without trailing punctuation.
+
 ===============================================================================
                                                               *shell-background*
 Background ~

plugin/shell.vim

@@ -1,6 +1,6 @@
 " Vim plug-in
 " Author: Peter Odding <peter@peterodding.com>
-" Last Change: September 18, 2011
+" Last Change: October 28, 2011
 " URL: http://peterodding.com/code/vim/shell/
 
 " Support for automatic update using the GLVS plug-in.
@@ -18,6 +18,13 @@ if !exists('g:shell_mappings_enabled')
   let g:shell_mappings_enabled = 1
 endif
 
+if !exists('g:shell_verify_urls')
+  " Set this to true if your URLs include significant trailing punctuation and
+  " your Vim is compiled with Python support. XXX In this case the shell
+  " plug-in will perform HTTP HEAD requests on your behalf.
+  let g:shell_verify_urls = 0
+endif
+
 " Automatic commands. {{{1
 
 augroup PluginShell