Skip to content

Commit

Permalink
Revert "Simplify core.http_request"
Browse files Browse the repository at this point in the history
This reverts commit c53a768 (for now!).
It seems to break local obs instances (see issue #202) (this needs
further debugging). Moreover, it breaks the python 3.4 - excerpt
from a travis run:

======================================================================
ERROR: test_added_missing2 (test_commit.TestCommit)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/opt/python/3.4.2/lib/python3.4/urllib/request.py", line 1111, in do_request_
    mv = memoryview(data)
TypeError: memoryview: _io.BufferedReader object does not have the buffer interface

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/travis/build/openSUSE/osc/tests/common.py", line 122, in wrapped_test_method
    test_method(*args)
  File "/home/travis/build/openSUSE/osc/tests/common.py", line 122, in wrapped_test_method
    test_method(*args)
  File "/home/travis/build/openSUSE/osc/tests/common.py", line 122, in wrapped_test_method
    test_method(*args)
  File "/home/travis/build/openSUSE/osc/tests/common.py", line 122, in wrapped_test_method
    test_method(*args)
  File "/home/travis/build/openSUSE/osc/tests/common.py", line 122, in wrapped_test_method
    test_method(*args)
  File "/home/travis/build/openSUSE/osc/tests/test_commit.py", line 290, in test_added_missing2
    p.commit()
  File "/home/travis/build/openSUSE/osc/tests/osc/core.py", line 1471, in commit
    self.put_source_file(filename, tdir)
  File "/home/travis/build/openSUSE/osc/tests/osc/core.py", line 1319, in put_source_file
    http_PUT(u, file = tfilename)
  File "/home/travis/build/openSUSE/osc/tests/osc/core.py", line 3243, in http_PUT
    def http_PUT(*args, **kwargs):    return http_request('PUT', *args, **kwargs)
  File "/home/travis/build/openSUSE/osc/tests/osc/core.py", line 3231, in http_request
    fd = urlopen(req, data=data)
  File "/opt/python/3.4.2/lib/python3.4/urllib/request.py", line 153, in urlopen
    return opener.open(url, data, timeout)
  File "/opt/python/3.4.2/lib/python3.4/urllib/request.py", line 453, in open
    req = meth(req)
  File "/opt/python/3.4.2/lib/python3.4/urllib/request.py", line 1116, in do_request_
    data))
ValueError: Content-Length should be specified for iterable data of type <class '_io.BufferedReader'> <_io.BufferedReader name='/tmp/osc_test571whun4/osctest/added_missing/.osc/_in_commit/bar'>
  • Loading branch information
marcus-h committed Jun 2, 2016
1 parent 2364a08 commit acbd2c1
Showing 1 changed file with 39 additions and 11 deletions.
50 changes: 39 additions & 11 deletions osc/core.py
Expand Up @@ -3188,8 +3188,15 @@ def makeurl(baseurl, l, query=[]):
def http_request(method, url, headers={}, data=None, file=None):
"""wrapper around urllib2.urlopen for error handling,
and to support additional (PUT, DELETE) methods"""
if data is not None and file is not None:
raise ValueError('data and file are mutually exclusive')
def create_memoryview(obj):
if sys.version_info < (2, 7, 99):
# obj might be a mmap and python 2.7's mmap does not
# behave like a bytearray (a bytearray in turn can be used
# to create the memoryview). For now simply return a buffer
return buffer(obj)
return memoryview(obj)

filefd = None

if conf.config['http_debug']:
print('\n\n--', method, url, file=sys.stderr)
Expand All @@ -3215,25 +3222,46 @@ def http_request(method, url, headers={}, data=None, file=None):
if method == 'PUT' or (method == 'POST' and (data or file)):
req.add_header('Content-Type', 'application/octet-stream')

for i in headers.keys():
req.add_header(i, headers[i])
if isinstance(headers, type({})):
for i in headers.keys():
print(headers[i])
req.add_header(i, headers[i])

if file is not None and not req.has_header('Content-length'):
req.add_header('Content-length', os.path.getsize(file))
if file and not data:
size = os.path.getsize(file)
if size < 1024*512:
data = open(file, 'rb').read()
else:
import mmap
filefd = open(file, 'rb')
try:
if sys.platform[:3] != 'win':
data = mmap.mmap(filefd.fileno(), os.path.getsize(file), mmap.MAP_SHARED, mmap.PROT_READ)
else:
data = mmap.mmap(filefd.fileno(), os.path.getsize(file))
data = create_memoryview(data)
except EnvironmentError as e:
if e.errno == 19:
sys.exit('\n\n%s\nThe file \'%s\' could not be memory mapped. It is ' \
'\non a filesystem which does not support this.' % (e, file))
elif hasattr(e, 'winerror') and e.winerror == 5:
# falling back to the default io
data = open(file, 'rb').read()
else:
raise

if conf.config['debug']: print(method, url, file=sys.stderr)

try:
if file is not None:
data = open(file, 'rb')
elif isinstance(data, str):
if isinstance(data, str):
data = bytes(data, "utf-8")
fd = urlopen(req, data=data)

finally:
if hasattr(conf.cookiejar, 'save'):
conf.cookiejar.save(ignore_discard=True)
if file is not None:
data.close()

if filefd: filefd.close()

return fd

Expand Down

0 comments on commit acbd2c1

Please sign in to comment.