Better logging and more testing for curl staging.

pulsar/client/transport/curl.py

@@ -7,27 +7,31 @@
     from pycurl import Curl, HTTP_CODE
 except ImportError:
     curl_available = False
-from os.path import getsize
+
+import os.path
 
 
 PYCURL_UNAVAILABLE_MESSAGE = \
     "You are attempting to use the Pycurl version of the Pulsar client but pycurl is unavailable."
 
+NO_SUCH_FILE_MESSAGE = "Attempt to post file %s to URL %s, but file does not exist."
+POST_FAILED_MESSAGE = "Failed to post_file properly for url %s, remote server returned status code of %s."
+GET_FAILED_MESSAGE = "Failed to get_file properly for url %s, remote server returned status code of %s."
+
 
 class PycurlTransport(object):
 
     def execute(self, url, method=None, data=None, input_path=None, output_path=None):
         buf = _open_output(output_path)
         try:
-            c = _new_curl_object()
-            c.setopt(c.URL, url.encode('ascii'))
+            c = _new_curl_object_for_url(url)
             c.setopt(c.WRITEFUNCTION, buf.write)
             if method:
                 c.setopt(c.CUSTOMREQUEST, method)
             if input_path:
                 c.setopt(c.UPLOAD, 1)
                 c.setopt(c.READFUNCTION, open(input_path, 'rb').read)
-                filesize = getsize(input_path)
+                filesize = os.path.getsize(input_path)
                 c.setopt(c.INFILESIZE, filesize)
             if data:
                 c.setopt(c.POST, 1)
@@ -42,28 +46,29 @@ def execute(self, url, method=None, data=None, input_path=None, output_path=None
 
 
 def post_file(url, path):
-    c = _new_curl_object()
-    c.setopt(c.URL, url.encode('ascii'))
+    if not os.path.exists(path):
+        # pycurl doesn't always produce a great exception for this,
+        # wrap it in a better one.
+        message = NO_SUCH_FILE_MESSAGE % (path, url)
+        raise Exception(message)
+    c = _new_curl_object_for_url(url)
     c.setopt(c.HTTPPOST, [("file", (c.FORM_FILE, path.encode('ascii')))])
     c.perform()
     status_code = c.getinfo(HTTP_CODE)
     if int(status_code) != 200:
-        template = "Failed to post_file properly for url %s, remote server returned status code of %s."
-        message = template % (url, status_code)
+        message = POST_FAILED_MESSAGE % (url, status_code)
         raise Exception(message)
 
 
 def get_file(url, path):
     buf = _open_output(path)
     try:
-        c = _new_curl_object()
-        c.setopt(c.URL, url.encode('ascii'))
+        c = _new_curl_object_for_url(url)
         c.setopt(c.WRITEFUNCTION, buf.write)
         c.perform()
         status_code = c.getinfo(HTTP_CODE)
         if int(status_code) != 200:
-            template = "Failed to get_file properly for url %s, remote server returned status code of %s."
-            message = template % (url, status_code)
+            message = GET_FAILED_MESSAGE % (url, status_code)
             raise Exception(message)
     finally:
         buf.close()
@@ -73,6 +78,12 @@ def _open_output(output_path):
     return open(output_path, 'wb') if output_path else StringIO()
 
 
+def _new_curl_object_for_url(url):
+    c = _new_curl_object()
+    c.setopt(c.URL, url.encode('ascii'))
+    return c
+
+
 def _new_curl_object():
     try:
         return Curl()

test/client_transport_test.py

@@ -2,6 +2,8 @@
 
 from pulsar.client.transport.standard import Urllib2Transport
 from pulsar.client.transport.curl import PycurlTransport
+from pulsar.client.transport.curl import post_file
+from pulsar.client.transport.curl import get_file
 from pulsar.client.transport import get_transport
 from tempfile import NamedTemporaryFile
 from .test_utils import files_server
@@ -35,6 +37,57 @@ def _test_transport(transport):
         assert open(output_path, 'r').read().find("Test123") >= 0
 
 
+def test_curl_put_get():
+    with files_server() as (server, directory):
+        server_url = server.application_url
+        path = os.path.join(directory, "test_for_GET")
+        request_url = u"%s?path=%s" % (server_url, path)
+
+        input = os.path.join(directory, "input")
+        output = os.path.join(directory, "output")
+        open(input, "wb").write(u"helloworld")
+
+        post_file(request_url, input)
+        get_file(request_url, output)
+        assert open(output, "rb").read() == u"helloworld"
+
+
+def test_curl_status_code():
+    with files_server() as (server, directory):
+        server_url = server.application_url
+        path = os.path.join(directory, "test_for_GET")
+        request_url = u"%s?path=%s" % (server_url, path)
+        # Verify curl doesn't just silently swallow errors.
+        exception_raised = False
+        try:
+            get_file(request_url, os.path.join(directory, "test"))
+        except Exception:
+            exception_raised = True
+        assert exception_raised
+
+        post_request_url = u"%s?path=%s" % (server_url, "/usr/bin/cow")
+        exception_raised = False
+        try:
+            post_file(post_request_url, os.path.join(directory, "test"))
+        except Exception:
+            exception_raised = True
+        assert exception_raised
+
+
+def test_curl_problems():
+    with files_server() as (server, directory):
+        server_url = server.application_url
+        path = os.path.join(directory, "test_for_GET")
+        request_url = u"%s?path=%s" % (server_url, path)
+        exception_raised = False
+        try:
+            # Valid destination but the file to post doesn't exist.
+            post_file(request_url, os.path.join(directory, "test"))
+        except Exception:
+            exception_raised = True
+        assert exception_raised
+
+
 def test_get_transport():
     assert type(get_transport(None, FakeOsModule("1"))) == PycurlTransport
     assert type(get_transport(None, FakeOsModule("TRUE"))) == PycurlTransport