Merge from master (#78)

* Adding yaml for Shippable CI.
* Fix for pip 10 which misses .main()
* Adding `recv_into` support (#72)
* Adding socket.recv_into() plus its test (#71).
* Forcing `pip` upgrade.
* Fix for #65, thanks to the @jmcbailey patch at gabrielfalcao/HTTPretty#242.
* Improving the check, removing dependency of `requests` which is only acting as wrapper.
* Increasing stability of `true_sendall` function.
* Improving the `flake8` usage.
* Minor changes.
* Status badge from Shippable

.gitignore

@@ -25,3 +25,5 @@ Documentation/
 doc/build
 .buildinfo
 .coverage
+shippable
+.pytest_cache/

.travis.yml

@@ -3,13 +3,12 @@ services:
 language: python
 python:
   - "2.7"
-  - "3.3"
   - "3.4"
   - "3.5"
   - "3.6"
-  # - "3.7"
+#  - "3.7-dev"  # need to investigate the failure
   - "pypy"
-  - "pypy3"
+#  - "pypy3"
 install:
   - make develop
 script:

Makefile

@@ -4,6 +4,7 @@ install-dev-requirements:
 	pip install -U -q -e .
 
 install-test-requirements:
+	pip install -U pip
 	pip install -U -q -e .[tests]
 
 test-python:
@@ -13,13 +14,19 @@ test-python:
 
 lint-python:
 	@echo "Linting Python files"
-	flake8 --exit-zero --ignore=E501 --exclude=.git,compat.py mocket
+	flake8 --ignore=E501,E731 --exclude=.git,compat.py mocket
 	@echo ""
 
 develop: install-test-requirements install-dev-requirements
+	mkdir -p shippable/testresults
+	mkdir -p shippable/codecoverage
 
 test: install-test-requirements lint-python test-python
 
+test-ci: install-test-requirements lint-python
+	python runtests.py --junitxml=shippable/testresults/nosetests.xml \
+	--cov-report=xml:shippable/codecoverage/coverage.xml
+
 safetest:
 	export SKIP_TRUE_REDIS=1; export SKIP_TRUE_HTTP=1; make test
 
@@ -33,5 +40,5 @@ clean:
 	rm -rf *.egg-info
 	find . -type d -name __pycache__ -exec rm -rf {} \;
 
-.PHONY: publish clean
+.PHONY: clean publish safetest test test-ci develop lint-python test-python install-test-requirements install-dev-requirements
 

README.rst

@@ -2,8 +2,8 @@
 mocket /mɔˈkɛt/
 ===============
 
-.. image:: https://api.travis-ci.org/mindflayer/python-mocket.svg?branch=master
-    :target: http://travis-ci.org/mindflayer/python-mocket
+.. image:: https://api.shippable.com/projects/5af070d0a55fb8070034316f/badge?branch=master
+    :target: https://app.shippable.com/github/mindflayer/python-mocket
 
 .. image:: https://coveralls.io/repos/github/mindflayer/python-mocket/badge.svg?branch=master
     :target: https://coveralls.io/github/mindflayer/python-mocket?branch=master

mocket/__init__.py

@@ -7,4 +7,4 @@
 
 __all__ = (mocketize, Mocket, MocketEntry, Mocketizer)
 
-__version__ = '2.2.3'
+__version__ = '2.4.0'

mocket/mocket.py

@@ -27,6 +27,13 @@
     JSONDecodeError,
 )
 
+try:
+    from urllib3.contrib.pyopenssl import inject_into_urllib3, extract_from_urllib3
+    pyopenssl_override = True
+except ImportError:
+    pyopenssl_override = False
+
+
 __all__ = (
     'true_socket',
     'true_create_connection',
@@ -131,7 +138,7 @@ def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, **kw
         self.true_ssl_socket = wrap_ssl_socket(true_ssl_socket, true_socket(family, type, proto), true_ssl_context())
         self.fd = MocketSocketCore()
         self._connected = False
-        self._buflen = 65536
+        self._buflen = 4096
         self._entry = None
         self.family = int(family)
         self.type = int(type)
@@ -246,6 +253,9 @@ def sendall(self, data, entry=None, *args, **kwargs):
     def read(self, buffersize):
         return self.fd.read(buffersize)
 
+    def recv_into(self, buffer, buffersize, flags=None):
+        return buffer.write(self.fd.read(buffersize))
+
     def recv(self, buffersize, flags=None):
         if Mocket.r_fd and Mocket.w_fd:
             return os.read(Mocket.r_fd, buffersize)
@@ -297,25 +307,16 @@ def true_sendall(self, data, *args, **kwargs):
 
             self.true_socket.connect((host, port))
             self.true_socket.sendall(data, *args, **kwargs)
-            encoded_response = b''
-
-            while not encoded_response:
-                # https://github.com/kennethreitz/requests/blob/master/tests/testserver/server.py#L13
-                while select.select([self.true_socket], [], [], 0.5)[0]:
-                    recv = self.true_socket.recv(self._buflen)
-                    if recv:
-                        encoded_response += recv
-                    else:
-                        break
-
-                if not encoded_response:
-                    self.true_ssl_socket.connect((host, port))
-                    while select.select([self.true_ssl_socket], [], [], 0.5)[0]:
-                        recv = self.true_ssl_socket.recv(self._buflen)
-                        if recv:
-                            encoded_response += recv
-                        else:
-                            break
+            encoded_response = None
+            # https://github.com/kennethreitz/requests/blob/master/tests/testserver/server.py#L13
+            while True:
+                more_to_read = select.select([self.true_socket], [], [], 0.5)[0]
+                if not more_to_read and encoded_response is not None:
+                    break
+                recv = self.true_socket.recv(self._buflen)
+                if not recv and encoded_response is not None:
+                    break
+                encoded_response = encoded_response or b'' + recv
 
             # dump the resulting dictionary to a JSON file
             if Mocket.get_truesocket_recording_dir():
@@ -410,6 +411,9 @@ def enable(namespace=None, truesocket_recording_dir=None):
             '\x7f\x00\x00\x01',
             'utf-8'
         )
+        if pyopenssl_override:
+            # Take out the pyopenssl version - use the default implementation
+            extract_from_urllib3()
 
     @staticmethod
     def disable():
@@ -425,6 +429,9 @@ def disable():
         ssl.SSLContext = ssl.__dict__['SSLContext'] = true_ssl_context
         socket.inet_pton = socket.__dict__['inet_pton'] = true_inet_pton
         Mocket.reset()
+        if pyopenssl_override:
+            # Put the pyopenssl version back in place
+            inject_into_urllib3()
 
     @classmethod
     def get_namespace(cls):

mocket/mockredis.py

@@ -20,21 +20,26 @@ class Redisizer(byte_type):
     @staticmethod
     def tokens(iterable):
         iterable = [encode_to_bytes(x) for x in iterable]
-        return ['*{0}'.format(len(iterable)).encode('utf-8')] + list(chain(*zip(['${0}'.format(len(x)).encode('utf-8') for x in iterable], iterable)))
+        return [
+            '*{0}'.format(len(iterable)).encode('utf-8')
+        ] + list(
+            chain(*zip(['${0}'.format(len(x)).encode('utf-8') for x in iterable], iterable))
+        )
 
     @staticmethod
     def redisize(data):
+        def get_conversion(t):
+            return {
+                dict: lambda x: b'\r\n'.join(Redisizer.tokens(list(chain(*tuple(x.items()))))),
+                int: lambda x: ':{0}'.format(x).encode('utf-8'),
+                text_type: lambda x: '${0}\r\n{1}'.format(len(x.encode('utf-8')), x).encode('utf-8'),
+                list: lambda x: b'\r\n'.join(Redisizer.tokens(x)),
+            }[t]
         if isinstance(data, Redisizer):
             return data
         if isinstance(data, byte_type):
             data = decode_from_bytes(data)
-        CONVERSION = {
-            dict: lambda x: b'\r\n'.join(Redisizer.tokens(list(chain(*tuple(x.items()))))),
-            int: lambda x: ':{0}'.format(x).encode('utf-8'),
-            text_type: lambda x: '${0}\r\n{1}'.format(len(x.encode('utf-8')), x).encode('utf-8'),
-            list: lambda x: b'\r\n'.join(Redisizer.tokens(x)),
-        }
-        return Redisizer(CONVERSION[type(data)](data) + b'\r\n')
+        return Redisizer(get_conversion(data.__class__)(data) + b'\r\n')
 
     @staticmethod
     def command(description, _type='+'):

mocket/plugins/httpretty/core.py

@@ -1,3 +1,3 @@
-from mocket.compat import text_type, byte_type, decode_from_bytes
+from mocket.compat import decode_from_bytes
 
 decode_utf8 = decode_from_bytes

mocket/plugins/pook_mock_engine.py

@@ -67,4 +67,3 @@ def mocket_mock_fun(*args, **kwargs):
         # mocking pook.mock()
         self.pook_mock_fun = self.engine.mock
         self.engine.mock = mocket_mock_fun
-

requirements.txt

@@ -1,4 +1,5 @@
 python-magic
 six
 decorator
-hexdump
+hexdump
+urllib3

runtests.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 import sys
+import os
 
 
 def runtests(args=None):
     import pytest
-    import pip
 
     if not args:
         args = []
@@ -17,7 +17,7 @@ def runtests(args=None):
     if major == 3 and minor >= 5:
         python35 = True
 
-        pip.main(['install', 'aiohttp', 'async_timeout'])
+        os.system('pip install aiohttp async_timeout')
 
     if not any(a for a in args[1:] if not a.startswith('-')):
         args.append('tests/main')

shippable.yaml

@@ -0,0 +1,18 @@
+language: python
+
+python:
+  - 2.7
+  - 3.4
+  - 3.5
+  - 3.6
+  #- 3.7
+  - pypy
+  - pypy3
+
+services:
+  - redis
+
+build:
+  ci:
+    - shippable_retry make develop
+    - make test-ci

test_requirements.txt

@@ -5,4 +5,4 @@ mock
 requests
 redis
 gevent
-sure
+sure

tests/main/test_https.py

@@ -41,6 +41,7 @@ def test_json(response):
 recording_directory = tempfile.mkdtemp()
 
 
+@pytest.mark.skipif('os.getenv("SKIP_TRUE_HTTP", False)')
 @mocketize(truesocket_recording_dir=recording_directory)
 def test_truesendall_with_recording_https():
     url = 'https://httpbin.org/ip'

tests/main/test_mocket.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 import socket
 from unittest import TestCase
+import io
 
 import pytest
 
@@ -94,6 +95,28 @@ def test_subsequent_recv_requests_have_correct_length(self):
             assert _so.recv(4096) == b'Short'
             _so.close()
 
+    def test_recv_into(self):
+        Mocket.register(
+            MocketEntry(
+                ('localhost', 80),
+                [
+                    b'Long payload',
+                    b'Short'
+                ]
+            )
+        )
+        buffer = io.BytesIO()
+        with Mocketizer():
+            _so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            _so.connect(('localhost', 80))
+            _so.sendall(b'first\r\n')
+            assert _so.recv_into(buffer, 4096) == 12
+            _so.sendall(b'second\r\n')
+            assert _so.recv_into(buffer, 4096) == 5
+            _so.close()
+        buffer.seek(0)
+        assert buffer.read() == b'Long payloadShort'
+
 
 class MocketizeTestCase(TestCase):
     def mocketize_setup(self):