Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

working on faster wsgi server

  • Loading branch information...
commit ba4a176282c55fb3f8fa9ad2efe5a17f32667b6e 1 parent 4e3cdec
authored
6  examples/http.py
... ...
@@ -1,5 +1,8 @@
  1
+import logging
  2
+
1 3
 from concurrence import dispatch
2  
-from concurrence.http import WSGIServer
  4
+#from concurrence.http import WSGIServer
  5
+from concurrence.http.server2 import WSGIServer
3 6
 
4 7
 def hello_world(environ, start_response):
5 8
     start_response("200 OK", [])
@@ -10,4 +13,5 @@ def main():
10 13
     server.serve(('localhost', 8080))
11 14
 
12 15
 if __name__ == '__main__':
  16
+    logging.basicConfig()
13 17
     dispatch(main)
62  lib/concurrence/database/mysql/__init__.py
@@ -30,9 +30,9 @@ class COMMAND:
30 30
     QUERY = 0x03
31 31
     LIST = 0x04
32 32
     PING = 0x0e
33  
-    
  33
+
34 34
 class CAPS(object):
35  
-    LONG_PASSWORD =   1   # new more secure passwords 
  35
+    LONG_PASSWORD =   1   # new more secure passwords
36 36
     FOUND_ROWS = 2    #Found instead of affected rows
37 37
     LONG_FLAG = 4    #Get all column flags */
38 38
     CONNECT_WITH_DB = 8  # One can specify db on connect */
@@ -51,7 +51,7 @@ class CAPS(object):
51 51
     MULTI_STATEMENTS= 65536   # Enable/disable multi-stmt support */
52 52
     MULTI_RESULTS   = 131072  # Enable/disable multi-results */
53 53
 
54  
-    __ALL__ = {LONG_PASSWORD: 'CLIENT_LONG_PASSWORD', 
  54
+    __ALL__ = {LONG_PASSWORD: 'CLIENT_LONG_PASSWORD',
55 55
                FOUND_ROWS: 'CLIENT_FOUND_ROWS',
56 56
                LONG_FLAG: 'CLIENT_LONG_FLAG',
57 57
                CONNECT_WITH_DB: 'CLIENT_CONNECT_WITH_DB',
@@ -69,7 +69,7 @@ class CAPS(object):
69 69
                SECURE_CONNECTION: 'CLIENT_SECURE_CONNECTION',
70 70
                MULTI_STATEMENTS: 'CLIENT_MULTI_STATEMENTS',
71 71
                MULTI_RESULTS: 'CLIENT_MULTI_RESULTS'}
72  
- 
  72
+
73 73
     @classmethod
74 74
     def dbg(cls, caps):
75 75
         for value, name in cls.__ALL__.items():
@@ -80,7 +80,7 @@ def create_scramble_buff():
80 80
     import random
81 81
     return ''.join([chr(random.randint(0, 255)) for _ in xrange(20)])
82 82
 
83  
-        
  83
+
84 84
 class BufferedPacketWriter(BufferedWriter):
85 85
     #TODO make writers really buffered
86 86
     def __init__(self, stream, buffer):
@@ -94,16 +94,16 @@ def write_error(self, errno, errmsg):
94 94
         self.buffer.write_byte((errno >> 8) & 0xFF)
95 95
         #ERROR MSG:
96 96
         self.buffer.write_bytes(self.ERROR_TEMPLATE % errmsg)
97  
-        
  97
+
98 98
     def write_ok(self, field_count, affected_rows, insert_id, server_status, warning_count, msg = ''):
99  
-        self.buffer.write_byte(field_count)  
  99
+        self.buffer.write_byte(field_count)
100 100
         self.buffer.write_byte(affected_rows)
101  
-        self.buffer.write_byte(insert_id) 
  101
+        self.buffer.write_byte(insert_id)
102 102
         self.buffer.write_short(server_status) #server Status
103  
-        self.buffer.write_short(warning_count) 
  103
+        self.buffer.write_short(warning_count)
104 104
         if msg:
105 105
             self.buffer.write_bytes(msg)
106  
-      
  106
+
107 107
     def write_greeting(self, scramble_buff, protocol_version, server_version, thread_id, server_caps, server_language, server_status):
108 108
 
109 109
         self.buffer.write_byte(protocol_version)
@@ -116,35 +116,35 @@ def write_greeting(self, scramble_buff, protocol_version, server_version, thread
116 116
         self.buffer.write_short(server_status)
117 117
         self.buffer.write_bytes('\0' * 13) #filler
118 118
         self.buffer.write_bytes(scramble_buff[8:])
119  
-        
  119
+
120 120
     def write_header(self, length, packet_number):
121 121
         self.buffer.write_int((length - 4) | (packet_number << 24))
122  
-        
  122
+
123 123
     def start(self):
124 124
         """starts building a packet"""
125 125
         self.start_position = self.buffer.position #remember start of header
126 126
         self.buffer.skip(4) #reserve room for header
127  
-        
  127
+
128 128
     def finish(self, packet_number):
129 129
         """finishes packet by going back to start of packet and writing header and packetNumber"""
130 130
         position = self.buffer.position
131  
-        length = self.buffer.position - self.start_position 
  131
+        length = self.buffer.position - self.start_position
132 132
         #print length
133 133
         self.buffer.position = self.start_position
134 134
         self.write_header(length, packet_number)
135 135
         self.buffer.position = position
136  
-        
  136
+
137 137
     def write_int(self, i):
138 138
         self.buffer.write_int(i)
139 139
 
140 140
     def write_lcb(self, b):
141 141
         assert b < 128, "TODO larger numbers"
142 142
         self.buffer.write_byte(b)
143  
-        
  143
+
144 144
     def write_lcs(self, s):
145 145
         self.write_lcb(len(s))
146 146
         self.buffer.write_bytes(s)
147  
-        
  147
+
148 148
 
149 149
 class BufferedPacketReader(BufferedReader):
150 150
     def __init__(self, stream, buffer):
@@ -155,34 +155,34 @@ def __init__(self, stream, buffer):
155 155
 
156 156
     def read_packets(self):
157 157
         reader = self.reader
158  
-        
  158
+
159 159
         READ_RESULT_END = PACKET_READ_RESULT.END
160 160
         READ_RESULT_MORE = PACKET_READ_RESULT.MORE
161 161
 
162 162
         while True:
163 163
             read_result = reader.read_packet()
164 164
             if read_result & READ_RESULT_END:
165  
-                yield reader.packet                    
  165
+                yield reader.packet
166 166
             if not (read_result & READ_RESULT_MORE):
167  
-                self._read_more()
168  
- 
  167
+                self.fill()
  168
+
169 169
     def read_packet(self):
170 170
         return self.read_packets().next()
171 171
 
172 172
     def read_length_coded_binary(self):
173 173
         return self.reader.read_length_coded_binary()
174  
-    
  174
+
175 175
     def read_fields(self, field_count):
176  
-        
  176
+
177 177
         #generator for rest of result packets
178 178
         packets = self.read_packets()
179  
-        
  179
+
180 180
         #read field types
181 181
         fields = []
182  
-        
  182
+
183 183
         reader = self.reader
184 184
         i = 0
185  
-        while i < field_count:            
  185
+        while i < field_count:
186 186
             _ = packets.next()
187 187
             fields.append(reader.read_field_type())
188 188
             i += 1
@@ -190,12 +190,12 @@ def read_fields(self, field_count):
190 190
         #end of field types
191 191
         packet = packets.next()
192 192
         assert packet.read_byte() == 0xFE, "expected end of fields"
193  
-        
194  
-        return fields 
  193
+
  194
+        return fields
195 195
 
196 196
     def read_rows(self, fields, row_count = 100):
197 197
         reader = self.reader
198  
-        
  198
+
199 199
         READ_RESULT_EOF = PACKET_READ_RESULT.EOF
200 200
         READ_RESULT_MORE = PACKET_READ_RESULT.MORE
201 201
 
@@ -206,6 +206,6 @@ def read_rows(self, fields, row_count = 100):
206 206
             if read_result & READ_RESULT_EOF:
207 207
                 break
208 208
             if not (read_result & READ_RESULT_MORE):
209  
-                self._read_more()
  209
+                self.fill()
  210
+
210 211
 
211  
-            
10  lib/concurrence/http/__init__.py
@@ -3,11 +3,13 @@
3 3
 # This module is part of the Concurrence Framework and is released under
4 4
 # the New BSD License: http://www.opensource.org/licenses/bsd-license.php
5 5
 
  6
+from concurrence.http._http import HTTPParser, HTTPParserError
  7
+
6 8
 class HTTPError(Exception): pass
7 9
 
8 10
 class HTTPRequest(object):
9 11
     """A class representing a HTTP request."""
10  
-    
  12
+
11 13
     def __init__(self, path = None, method = None, host = None):
12 14
         """Create a new http request for *path* using *method* to *host*."""
13 15
         self.path = path
@@ -17,11 +19,11 @@ def __init__(self, path = None, method = None, host = None):
17 19
         self._body = None
18 20
 
19 21
     def add_header(self, key, value):
20  
-        """Adds a new header to the request with name *key* and given *value*."""        
  22
+        """Adds a new header to the request with name *key* and given *value*."""
21 23
         self.headers.append((key, value))
22 24
 
23 25
     def _set_body(self, body):
24  
-        if body is not None: 
  26
+        if body is not None:
25 27
             assert type(body) == str
26 28
             self.add_header('Content_length', len(body))
27 29
         self._body = body
@@ -33,7 +35,7 @@ def _get_body(self):
33 35
 
34 36
 class HTTPResponse(object):
35 37
     """Represents a HTTP Response."""
36  
-    
  38
+
37 39
     def __init__(self):
38 40
         self.headers = []
39 41
         self.status = ''
122  lib/concurrence/http/concurrence.http._http.pyx
@@ -4,6 +4,7 @@
4 4
 # the New BSD License: http://www.opensource.org/licenses/bsd-license.php
5 5
 
6 6
 from concurrence.io._io cimport Buffer
  7
+from concurrence.io._io import BufferUnderflowError
7 8
 
8 9
 cdef extern from "http11_parser.h":
9 10
     ctypedef struct http_parser
@@ -35,77 +36,106 @@ cdef extern from "Python.h":
35 36
 cdef class HTTPParser
36 37
 
37 38
 cdef void cb_request_method(void *data, char *at, size_t length):
38  
-    (<HTTPParser>data).request_method(PyString_FromStringAndSize(at, length))
  39
+    (<HTTPParser>data)._cb_request_method(PyString_FromStringAndSize(at, length))
39 40
 
40 41
 cdef void cb_request_uri(void *data, char *at, size_t length):
41  
-    (<HTTPParser>data).request_uri(PyString_FromStringAndSize(at, length))
  42
+    (<HTTPParser>data)._cb_request_uri(PyString_FromStringAndSize(at, length))
42 43
 
43 44
 cdef void cb_fragment(void *data, char *at, size_t length):
44  
-    (<HTTPParser>data).fragment(PyString_FromStringAndSize(at, length))
  45
+    #(<HTTPParser>data)._cb_fragment(PyString_FromStringAndSize(at, length))
  46
+    pass #unused for now, fragment is not part of cgi spec
45 47
 
46 48
 cdef void cb_request_path(void *data, char *at, size_t length):
47  
-    (<HTTPParser>data).request_path(PyString_FromStringAndSize(at, length))
  49
+    (<HTTPParser>data)._cb_request_path(PyString_FromStringAndSize(at, length))
48 50
 
49 51
 cdef void cb_query_string(void *data, char *at, size_t length):
50  
-    (<HTTPParser>data).query_string(PyString_FromStringAndSize(at, length))
  52
+    (<HTTPParser>data)._cb_query_string(PyString_FromStringAndSize(at, length))
51 53
 
52 54
 cdef void cb_http_version(void *data, char *at, size_t length):
53  
-    (<HTTPParser>data).http_version(PyString_FromStringAndSize(at, length))
  55
+    (<HTTPParser>data)._cb_http_version(PyString_FromStringAndSize(at, length))
54 56
 
55 57
 cdef void cb_header_done(void *data, char *at, size_t length):
56  
-    (<HTTPParser>data).header_done(PyString_FromStringAndSize(at, length))
  58
+    #(<HTTPParser>data)._cb_header_done(PyString_FromStringAndSize(at, length))
  59
+    pass #unused
57 60
 
58 61
 cdef void cb_field(void *data, char *field, size_t flen, char *value, size_t vlen):
59  
-    (<HTTPParser>data).field(PyString_FromStringAndSize(field, flen), PyString_FromStringAndSize(value, vlen))
  62
+    (<HTTPParser>data)._cb_field(PyString_FromStringAndSize(field, flen), PyString_FromStringAndSize(value, vlen))
  63
+
  64
+class HTTPParserError(Exception):
  65
+    pass
60 66
 
61 67
 cdef class HTTPParser:
62 68
     """
63 69
     """
64 70
     cdef http_parser *_parser
65 71
     cdef Buffer _buffer
  72
+    cdef readonly environ
66 73
 
67 74
     def __cinit__(self, Buffer buffer):
68  
-        self._buffer = buffer
69 75
         self._parser = http_parser_alloc(<void *>self, cb_request_method, cb_request_uri, cb_fragment, cb_request_path, cb_query_string, cb_http_version, cb_header_done, cb_field)
70 76
         http_parser_init(self._parser)
71 77
 
  78
+    def __init__(self, Buffer buffer):
  79
+        self._buffer = buffer
  80
+        self.environ = {}
  81
+
72 82
     def __dealloc__(self):
73 83
         http_parser_free(self._parser)
74 84
 
75  
-    cdef request_method(self, method):
76  
-        pass
77  
-        #print 'method:', repr(method)
78  
-
79  
-    cdef query_string(self, qs):
80  
-        pass
81  
-        #print 'qs:', repr(qs)
82  
-
83  
-    cdef request_path(self, path):
84  
-        pass
85  
-        #print 'path:', repr(path)
86  
-
87  
-
88  
-    cdef fragment(self, fragment):
89  
-        pass
90  
-        #print 'fragment', repr(fragment)
91  
-
92  
-    cdef request_uri(self, uri):
93  
-        pass
94  
-        #print 'r_uri', uri
95  
-
96  
-    cdef http_version(self, version):
97  
-        pass
98  
-        #print 'version', repr(version)
99  
-
100  
-    cdef header_done(self, hd):
101  
-        pass
102  
-        #print 'header_done', repr(hd)
103  
-
104  
-    cdef field(self, name, value):
105  
-        pass
106  
-        #print 'field', repr(name), repr(value)
107  
-
108  
-    def execute(self):
109  
-        return http_parser_execute(self._parser, <char *>self._buffer._buff, self._buffer._remaining(), self._buffer._position)
110  
-
111  
-
  85
+    cdef _cb_request_method(self, method):
  86
+        self.environ['REQUEST_METHOD'] = method
  87
+
  88
+    cdef _cb_query_string(self, qs):
  89
+        self.environ['QUERY_STRING'] = qs
  90
+
  91
+    cdef _cb_request_path(self, path):
  92
+        self.environ['PATH_INFO'] = path
  93
+
  94
+    cdef _cb_fragment(self, fragment):
  95
+        pass #unused, not part of cgi spec
  96
+
  97
+    cdef _cb_request_uri(self, uri):
  98
+        self.environ['REQUEST_URI'] = uri
  99
+
  100
+    cdef _cb_http_version(self, version):
  101
+        self.environ['HTTP_VERSION'] = version
  102
+
  103
+    cdef _cb_header_done(self, hd):
  104
+        pass #unused
  105
+
  106
+    cdef _cb_field(self, name, value):
  107
+        key = 'HTTP_' + name
  108
+        if key in self.environ:
  109
+            self.environ[key] += ',' + value # comma-separate multiple headers
  110
+        else:
  111
+            self.environ[key] = value
  112
+
  113
+    def parse(self):
  114
+        cdef size_t nread
  115
+        cdef int r
  116
+        if http_parser_is_finished(self._parser):
  117
+            raise HTTPParserError("cannot parse: parser already finished")
  118
+        elif http_parser_has_error(self._parser):
  119
+            raise HTTPParserError("cannot parse: parser already finished with error")
  120
+        remaining = self._buffer._remaining()
  121
+        if remaining > 0:
  122
+            nread = http_parser_execute(self._parser, <char *>self._buffer._buff, remaining, self._buffer._position)
  123
+            if http_parser_has_error(self._parser):
  124
+                raise HTTPParserError("parse error")
  125
+            else:
  126
+                self._buffer._position += self._buffer._position + nread
  127
+                return self.is_finished()
  128
+        else:
  129
+            raise BufferUnderflowError()
  130
+
  131
+    def is_finished(self):
  132
+        if http_parser_is_finished(self._parser):
  133
+            return True
  134
+        else:
  135
+            return False
  136
+
  137
+    def has_error(self):
  138
+        if http_parser_has_error(self._parser):
  139
+            return True
  140
+        else:
  141
+            return False
1  lib/concurrence/http/http11_parser_alloc.h
@@ -11,6 +11,7 @@ http_parser *http_parser_alloc(void *obj, element_cb request_method,
11 11
                                           element_cb http_version,
12 12
                                           element_cb header_done,
13 13
                                           field_cb field);
  14
+
14 15
 void http_parser_free(http_parser *);
15 16
 
16 17
 #endif
118  lib/concurrence/http/server.py
@@ -36,17 +36,17 @@ def __init__(self, request, reader):
36 36
         if content_length is None:
37 37
             self._channel = None
38 38
             self._n = None
39  
-            self._file = None        
  39
+            self._file = None
40 40
         else:
41 41
             self._n = int(content_length)
42 42
             self._file = reader.file()
43 43
             self._channel = Channel()
44  
-    
  44
+
45 45
     def _read_request_data(self):
46 46
         if self._n is not None:
47 47
             self._channel.receive() #wait till handler has read all input data
48 48
 
49  
-    def read(self, n):  
  49
+    def read(self, n):
50 50
         if self._n > 0:
51 51
             data = self._file.read(min(self._n, n))
52 52
             self._n -= len(data)
@@ -58,7 +58,7 @@ def read(self, n):
58 58
             return data
59 59
         else:
60 60
             return '' #EOF
61  
-        
  61
+
62 62
     def readline(self):
63 63
         assert False, 'TODO'
64 64
 
@@ -79,10 +79,10 @@ def writelines(self, s):
79 79
     def flush(self):
80 80
         assert False, 'TODO'
81 81
 
82  
-        
  82
+
83 83
 class WSGIRequest(object):
84 84
     log = logging.getLogger('WSGIRequest')
85  
-    
  85
+
86 86
     STATE_INIT = 0
87 87
     STATE_WAIT_FOR_REQUEST = 1
88 88
     STATE_READING_HEADER = 2
@@ -91,18 +91,18 @@ class WSGIRequest(object):
91 91
     STATE_WRITING_HEADER = 5
92 92
     STATE_WRITING_DATA = 6
93 93
     STATE_FINISHED = 7
94  
-    
  94
+
95 95
     _disallowed_application_headers = set(['Date', 'Server'])
96 96
 
97 97
     def __init__(self, server):
98  
-        self._server = server        
  98
+        self._server = server
99 99
         self.version = None #http version
100 100
         self.environ = {}
101 101
         self.response_headers = []
102 102
         self.status = httplib.NOT_FOUND #or internal server error?
103  
-        self.exc_info = None     
104  
-        self.state = self.STATE_INIT   
105  
-        
  103
+        self.exc_info = None
  104
+        self.state = self.STATE_INIT
  105
+
106 106
     def start_response(self, status, response_headers, exc_info = None):
107 107
         self.status = status
108 108
         self.response_headers = response_headers
@@ -113,52 +113,52 @@ def get_response_header(self, key):
113 113
             if _key == key:
114 114
                 return value
115 115
         return None
116  
-        
  116
+
117 117
     def get_request_header(self, key):
118 118
         http_key = 'HTTP_' + key.replace('-', '_').upper()
119 119
         return self.environ.get(http_key, None)
120 120
 
121 121
     def write_response(self, response, writer):
122 122
         self.state = self.STATE_WRITING_HEADER
123  
-        
  123
+
124 124
         if self.version == 'HTTP/1.0':
125 125
             chunked = False
126 126
         else:
127 127
             chunked = True
128 128
 
129 129
         writer.clear()
130  
-        
131  
-        writer.write_bytes("%s %s\r\n" % (self.version, self.status))        
  130
+
  131
+        writer.write_bytes("%s %s\r\n" % (self.version, self.status))
132 132
         for header_name, header_value in self.response_headers:
133 133
             if header_name in self._disallowed_application_headers: continue
134 134
             writer.write_bytes("%s: %s\r\n" % (header_name, header_value))
135 135
         writer.write_bytes("Date: %s\r\n" % rfc822.formatdate())
136  
-        writer.write_bytes("Server: %s\r\n" % SERVER_ID) 
  136
+        writer.write_bytes("Server: %s\r\n" % SERVER_ID)
137 137
 
138 138
         if chunked:
139 139
             writer.write_bytes("Transfer-Encoding: chunked\r\n")
140 140
         else:
141 141
             response = ''.join(response)
142 142
             writer.write_bytes("Content-length: %d\r\n" % len(response))
143  
-        
  143
+
144 144
         writer.write_bytes("\r\n")
145  
-    
  145
+
146 146
         self.state = self.STATE_WRITING_DATA
147  
-        
  147
+
148 148
         if chunked:
149 149
             for chunk in response:
150 150
                 writer.write_bytes("%x;\r\n" % len(chunk))
151 151
                 writer.write_bytes(chunk)
152 152
                 writer.write_bytes("\r\n")
153  
-                
  153
+
154 154
             writer.write_bytes("0\r\n\r\n")
155 155
         else:
156 156
             writer.write_bytes(response)
157 157
 
158 158
         writer.flush() #TODO use special header to indicate no flush needed
159  
-        
  159
+
160 160
         self.state = self.STATE_FINISHED
161  
-    
  161
+
162 162
     def handle_request(self, application):
163 163
         try:
164 164
             return application(self.environ, self.start_response)
@@ -167,7 +167,7 @@ def handle_request(self, application):
167 167
         except:
168 168
             self.log.exception("unhandled exception while handling request")
169 169
             return self._server.internal_server_error(self.environ, self.start_response)
170  
-      
  170
+
171 171
     def read_request_data(self):
172 172
         self.environ['wsgi.input']._read_request_data()
173 173
 
@@ -178,17 +178,17 @@ def read_request(self, reader):
178 178
     def _read_request(self, reader):
179 179
 
180 180
         self.state = self.STATE_WAIT_FOR_REQUEST
181  
-        
182  
-        #setup readline iterator        
  181
+
  182
+        #setup readline iterator
183 183
         lines = reader.read_lines()
184  
-        
  184
+
185 185
         #parse status line, this will block to read the first request line
186 186
         line = lines.next().split()
187  
-        
  187
+
188 188
         self.state = self.STATE_READING_HEADER
189  
-        
  189
+
190 190
         u = urlparse.urlparse(line[1])
191  
-        
  191
+
192 192
         self.method = line[0]
193 193
         if self.method not in ['GET', 'POST']:
194 194
             raise HTTPError('Unsupported method: %s' % self.method)
@@ -202,27 +202,27 @@ def _read_request(self, reader):
202 202
         self.environ['SCRIPT_NAME'] = '' #TODO
203 203
         self.environ['PATH_INFO'] = u[2]
204 204
         self.environ['QUERY_STRING'] = u[4]
205  
-        
  205
+
206 206
         self.environ['wsgi.url_scheme'] = 'http'
207 207
         self.environ['wsgi.multiprocess'] = False
208 208
         self.environ['wsgi.multithread'] = True
209 209
         self.environ['wsgi.run_once'] = False
210 210
         self.environ['wsgi.version'] = (1, 0)
211  
-        
  211
+
212 212
         #rest of request headers
213 213
         for line in lines:
214 214
             if not line: break
215  
-            key, value = line.split(': ')
  215
+            key, value = line.split(':', 1)
216 216
             key = key.replace('-', '_').upper()
217 217
             value = value.strip()
218  
-            
219  
-            http_key = 'HTTP_' + key 
  218
+
  219
+            http_key = 'HTTP_' + key
220 220
             if http_key in self.environ:
221 221
                 self.environ[http_key] += ',' + value # comma-separate multiple headers
222 222
             else:
223 223
                 self.environ[http_key] = value
224 224
 
225  
-        #wsgi complience 
  225
+        #wsgi complience
226 226
         if 'HTTP_CONTENT_LENGTH' in self.environ:
227 227
             self.environ['CONTENT_LENGTH'] = self.environ['HTTP_CONTENT_LENGTH']
228 228
 
@@ -232,7 +232,7 @@ def _read_request(self, reader):
232 232
         #setup required wsgi streams
233 233
         self.environ['wsgi.input'] = WSGIInputStream(self, reader)
234 234
         self.environ['wsgi.errors'] = WSGIErrorStream()
235  
-        
  235
+
236 236
         if not 'HTTP_HOST' in self.environ:
237 237
             if self.version == 'HTTP/1.0':
238 238
                 #ok in version 1.0, TODO what should host in wsgi environ be?
@@ -250,9 +250,9 @@ def _read_request(self, reader):
250 250
         self.environ['SERVER_NAME'] = host
251 251
         self.environ['SERVER_PORT'] = port
252 252
         self.environ['SERVER_PROTOCOL'] = self.version
253  
-        
  253
+
254 254
         self.state = self.STATE_REQUEST_READ
255  
-        
  255
+
256 256
 
257 257
 class HTTPHandler(object):
258 258
     log = logging.getLogger('HTTPHandler')
@@ -263,12 +263,12 @@ class MSG_RESPONSE_WRITTEN(Message): pass
263 263
     class MSG_REQUEST_READ(Message): pass
264 264
     class MSG_READ_ERROR(Message): pass
265 265
     class MSG_WRITE_ERROR(Message): pass
266  
-    
  266
+
267 267
     def __init__(self, server):
268  
-        self._server = server     
  268
+        self._server = server
269 269
         self._reque = ReorderQueue()
270 270
 
271  
-    def write_responses(self, control, stream):        
  271
+    def write_responses(self, control, stream):
272 272
         try:
273 273
             for msg, (request, response), kwargs in Tasklet.receive():
274 274
                 request.write_response(response, stream.writer)
@@ -281,12 +281,12 @@ def read_requests(self, control, stream):
281 281
         try:
282 282
             while True:
283 283
                 request = WSGIRequest(self._server)
284  
-                request.read_request(stream.reader)                
  284
+                request.read_request(stream.reader)
285 285
                 self.MSG_REQUEST_READ.send(control)(request, None)
286 286
                 request.read_request_data()
287 287
         except EOFError, e:
288 288
             if request.state == request.STATE_WAIT_FOR_REQUEST:
289  
-                pass #this is normal at the end of the http KA connection (client closes) 
  289
+                pass #this is normal at the end of the http KA connection (client closes)
290 290
         except IOError, e:
291 291
             if e.errno == 104 and request.state == request.STATE_WAIT_FOR_REQUEST:
292 292
                 pass #connection reset by peer while waiting for request
@@ -299,12 +299,12 @@ def read_requests(self, control, stream):
299 299
 
300 300
     def handle_request(self, control, request, application):
301 301
         response = self._server.handle_request(request, application)
302  
-        self.MSG_REQUEST_HANDLED.send(control)(request, response)       
  302
+        self.MSG_REQUEST_HANDLED.send(control)(request, response)
303 303
 
304 304
     def handle(self, socket, application):
305 305
         stream = BufferedStream(socket)
306 306
         #implements http1.1 keep alive handler
307  
-        #there are several concurrent tasks for each connection; 
  307
+        #there are several concurrent tasks for each connection;
308 308
         #1 for reading requests, 1 or more for handling requests and 1 for writing responses
309 309
         #the current task (the one created to handle the socket connection) is the controller task,
310 310
         #e.g. it coordinates the actions of it's children by message passing
@@ -322,18 +322,18 @@ def handle(self, socket, application):
322 322
         #4. control sends message to writer to start writing the response (MSG_WRITE_RESPONSE)
323 323
         #5. writer notififies control when response is wriiten (MSG_RESPONSE_WRITTEN)
324 324
 
325  
-        #control wait for msgs to arrive:        
  325
+        #control wait for msgs to arrive:
326 326
         for msg, (request, response), kwargs in Tasklet.receive():
327 327
             if msg.match(self.MSG_REQUEST_READ):
328 328
                 #we use reque to be able to send the responses back in the correct order later
329 329
                 self._reque.start(request)
330 330
                 Tasklet.new(self.handle_request, name = 'request_handler')(control, request, application)
331  
-                
  331
+
332 332
             elif msg.match(self.MSG_REQUEST_HANDLED):
333 333
                 #we use reque to retire (send out) the responses in the correct order
334 334
                 for request, response in self._reque.finish(request, response):
335 335
                     self.MSG_WRITE_RESPONSE.send(response_writer)(request, response)
336  
-                    
  336
+
337 337
             elif msg.match(self.MSG_RESPONSE_WRITTEN):
338 338
                 if request.version == 'HTTP/1.0':
339 339
                     break #no keep-alive support in http 1.0
@@ -345,22 +345,22 @@ def handle(self, socket, application):
345 345
                 break #stop and close the connection
346 346
             elif msg.match(self.MSG_WRITE_ERROR):
347 347
                 break #stop and close the connection
348  
-            else:   
  348
+            else:
349 349
                 assert False, "unexpected msg in control loop"
350 350
 
351 351
         #kill reader and writer
352 352
         #any outstanding request will continue, but will exit by themselves
353 353
         response_writer.kill()
354 354
         request_reader.kill()
355  
-   
  355
+
356 356
         #close our side of the socket
357 357
         stream.close()
358  
-        
  358
+
359 359
 class WSGIServer(object):
360 360
     """A HTTP/1.1 Web server with WSGI application interface.
361  
-    
362  
-    Usage:: 
363  
-    
  361
+
  362
+    Usage::
  363
+
364 364
         def hello_world(environ, start_response):
365 365
             start_response("200 OK", [])
366 366
             return ["<html>Hello, world!</html>"]
@@ -369,7 +369,7 @@ def hello_world(environ, start_response):
369 369
         server.serve(('localhost', 8080))
370 370
     """
371 371
     log = logging.getLogger('WSGIServer')
372  
-    
  372
+
373 373
     read_timeout = HTTP_READ_TIMEOUT
374 374
 
375 375
     def __init__(self, application, request_log_level = logging.DEBUG):
@@ -382,17 +382,17 @@ def internal_server_error(self, environ, start_response):
382 382
         """Default WSGI application for creating a default `500 Internal Server Error` response on any
383 383
         unhandled exception.
384 384
         The default response will render a traceback with a text/plain content-type.
385  
-        Can be overridden to provide a custom response."""           
  385
+        Can be overridden to provide a custom response."""
386 386
         start_response('500 Internal Server Error', [('Content-type', 'text/plain')])
387 387
         return [traceback.format_exc(20)]
388 388
 
389 389
     def handle_request(self, request, application):
390  
-        """All HTTP requests pass trough this method. 
  390
+        """All HTTP requests pass trough this method.
391 391
         This method provides a hook for logging, statistics and or further processing w.r.t. the *request*."""
392 392
         response = request.handle_request(application)
393 393
         self.log.log(self._request_log_level, "%s %s", request.status, request.uri)
394 394
         return response
395  
-        
  395
+
396 396
     def handle_connection(self, socket):
397 397
         """All HTTP connections pass trough this method.
398 398
         This method provides a hook for logging, statistics and or further processing w.r.t. the connection."""
@@ -401,5 +401,5 @@ def handle_connection(self, socket):
401 401
     def serve(self, endpoint):
402 402
         """Serves the application at the given *endpoint*. The *endpoint* must be a tuple (<host>, <port>)."""
403 403
         return Server.serve(endpoint, self.handle_connection)
404  
-                        
  404
+
405 405
 
104  lib/concurrence/http/server2.py
... ...
@@ -0,0 +1,104 @@
  1
+# Copyright (C) 2009, Hyves (Startphone Ltd.)
  2
+#
  3
+# This module is part of the Concurrence Framework and is released under
  4
+# the New BSD License: http://www.opensource.org/licenses/bsd-license.php
  5
+
  6
+#TODO write timeout
  7
+
  8
+from __future__ import with_statement
  9
+
  10
+import logging
  11
+import urlparse
  12
+import httplib
  13
+import traceback
  14
+import rfc822
  15
+
  16
+from concurrence import Tasklet, Message, Channel, TimeoutError, __version__
  17
+from concurrence.io import Socket, Buffer, BufferedStream
  18
+from concurrence.containers import ReorderQueue
  19
+from concurrence.timer import Timeout
  20
+from concurrence.http import HTTPError, HTTPParser
  21
+
  22
+SERVER_ID = "Concurrence-Http/%s" % __version__
  23
+
  24
+class HTTPConnection(object):
  25
+
  26
+    def __init__(self, server, client_socket):
  27
+        self._server = server
  28
+        self._stream = BufferedStream(client_socket)
  29
+
  30
+    def _write_response(self):
  31
+        response = "Hello World!"
  32
+        with self._stream.get_writer() as writer:
  33
+            writer.clear()
  34
+            writer.write_bytes("%s %s\r\n" % ('HTTP/1.1', '200 OK'))
  35
+            writer.write_bytes("Content-length: %d\r\n" % len(response))
  36
+            writer.write_bytes("\r\n")
  37
+            writer.write_bytes(response)
  38
+            writer.flush()
  39
+
  40
+    def _read_request(self):
  41
+
  42
+        with self._stream.get_reader() as reader:
  43
+            reader.fill() #initial fill
  44
+            parser = HTTPParser(reader.buffer)
  45
+            while True:
  46
+                #parse the buffer
  47
+                if parser.parse():
  48
+                    break #ok
  49
+                else:
  50
+                    reader.append() #extra fill, could not parse request with data currently in buffer
  51
+
  52
+        self._write_response()
  53
+        self._stream.close()
  54
+
  55
+    def handle(self):
  56
+        Tasklet.defer(self._read_request)
  57
+
  58
+class WSGIServer(object):
  59
+    """A HTTP/1.1 Web server with WSGI application interface.
  60
+
  61
+    Usage::
  62
+
  63
+        def hello_world(environ, start_response):
  64
+            start_response("200 OK", [])
  65
+            return ["<html>Hello, world!</html>"]
  66
+
  67
+        server = WSGIServer(hello_world)
  68
+        server.serve(('localhost', 8080))
  69
+    """
  70
+    log = logging.getLogger('WSGIServer')
  71
+
  72
+    def __init__(self, application, request_log_level = logging.DEBUG):
  73
+        """Create a new WSGIServer serving the given *application*. Optionally
  74
+        the *request_log_level* can be given. This loglevel is used for logging the requests."""
  75
+        self._application = application
  76
+        self._request_log_level = request_log_level
  77
+
  78
+    def internal_server_error(self, environ, start_response):
  79
+        """Default WSGI application for creating a default `500 Internal Server Error` response on any
  80
+        unhandled exception.
  81
+        The default response will render a traceback with a text/plain content-type.
  82
+        Can be overridden to provide a custom response."""
  83
+        start_response('500 Internal Server Error', [('Content-type', 'text/plain')])
  84
+        return [traceback.format_exc(20)]
  85
+
  86
+    def handle_request(self, request):
  87
+        """All HTTP requests pass trough this method.
  88
+        This method provides a hook for logging, statistics and or further processing w.r.t. the *request*."""
  89
+        response = request.handle_request(self._application)
  90
+        self.log.log(self._request_log_level, "%s %s", request.status, request.uri)
  91
+        return response
  92
+
  93
+    def handle_connection(self, client_socket):
  94
+        """All HTTP connections pass trough this method.
  95
+        This method provides a hook for logging, statistics and or further processing w.r.t. the connection."""
  96
+        HTTPConnection(self, client_socket).handle()
  97
+
  98
+    def serve(self, endpoint):
  99
+        """Serves the application at the given *endpoint*. The *endpoint* must be a tuple (<host>, <port>)."""
  100
+        server_socket = Socket.server(endpoint)
  101
+        for client_socket in server_socket.accept_iter():
  102
+            self.handle_connection(client_socket)
  103
+
  104
+
45  lib/concurrence/io/buffered.py
@@ -22,8 +22,23 @@ def file(self):
22 22
     def clear(self):
23 23
         self.buffer.clear()
24 24
 
25  
-    def _read_more(self):
26  
-        #any partially read data will be put in front, otherwise normal clear:
  25
+    def append(self):
  26
+        """tries to append data from the underlying stream to the current buffer."""
  27
+        if self.buffer.limit == self.buffer.capacity:
  28
+            raise BufferOverflowError("while appending")
  29
+        position, limit = self.buffer.position, self.buffer.limit
  30
+        self.buffer.position = limit
  31
+        self.buffer.limit = capacity
  32
+        if not self.stream.read(self.buffer, TIMEOUT_CURRENT):
  33
+            raise EOFError("while reading")
  34
+        #put position back where we found it, and set new limit
  35
+        self.buffer.limit = self.buffer.position
  36
+        self.buffer.position = position
  37
+
  38
+    def fill(self):
  39
+        """fills the readers buffer with data from the underlying stream.
  40
+        any partially read data still in the buffer will be moved to the front of the buffer.
  41
+        after reading from the stream, the buffer is flipped so that it can be read out by the reader."""
27 42
         self.buffer.compact()
28 43
         if not self.stream.read(self.buffer, TIMEOUT_CURRENT):
29 44
             raise EOFError("while reading")
@@ -32,26 +47,26 @@ def _read_more(self):
32 47
     def read_lines(self):
33 48
         """note that it cant read line accross buffer"""
34 49
         if self.buffer.remaining == 0:
35  
-            self._read_more()
  50
+            self.fill()
36 51
         while True:
37 52
             try:
38 53
                 yield self.buffer.read_line()
39 54
             except BufferUnderflowError:
40  
-                self._read_more()
  55
+                self.fill()
41 56
 
42 57
     def read_line(self):
43 58
         """note that it cant read line accross buffer"""
44 59
         if self.buffer.remaining == 0:
45  
-            self._read_more()
  60
+            self.fill()
46 61
         while True:
47 62
             try:
48 63
                 return self.buffer.read_line()
49 64
             except BufferUnderflowError:
50  
-                self._read_more()
  65
+                self.fill()
51 66
 
52 67
     def read_bytes_available(self):
53 68
         if self.buffer.remaining == 0:
54  
-            self._read_more()
  69
+            self.fill()
55 70
         return self.buffer.read_bytes(-1)
56 71
 
57 72
     def read_bytes(self, n):
@@ -64,27 +79,27 @@ def read_bytes(self, n):
64 79
                 s.append(buffer.read_bytes(min(n, r)))
65 80
                 n -= r
66 81
             else:
67  
-                self._read_more()
  82
+                self.fill()
68 83
 
69 84
         return ''.join(s)
70 85
 
71 86
     def read_int(self):
72 87
         if self.buffer.remaining == 0:
73  
-            self._read_more()
  88
+            self.fill()
74 89
         while True:
75 90
             try:
76 91
                 return self.buffer.read_int()
77 92
             except BufferUnderflowError:
78  
-                self._read_more()
  93
+                self.fill()
79 94
 
80 95
     def read_short(self):
81 96
         if self.buffer.remaining == 0:
82  
-            self._read_more()
  97
+            self.fill()
83 98
         while True:
84 99
             try:
85 100
                 return self.buffer.read_short()
86 101
             except BufferUnderflowError:
87  
-                self._read_more()
  102
+                self.fill()
88 103
 
89 104
 class BufferedWriter(object):
90 105
     def __init__(self, stream, buffer):
@@ -252,7 +267,7 @@ def readlines(self):
252 267
                 yield buffer.read_line(True)
253 268
             except BufferUnderflowError:
254 269
                 try:
255  
-                    reader._read_more()
  270
+                    reader.fill()
256 271
                 except EOFError:
257 272
                     buffer.flip()
258 273
                     yield buffer.read_bytes(-1)
@@ -268,7 +283,7 @@ def read(self, n = -1):
268 283
             while True:
269 284
                 s.append(buffer.read_bytes(-1))
270 285
                 try:
271  
-                    reader._read_more()
  286
+                    reader.fill()
272 287
                 except EOFError:
273 288
                     buffer.flip()
274 289
                     break
@@ -280,7 +295,7 @@ def read(self, n = -1):
280 295
                     n -= r
281 296
                 else:
282 297
                     try:
283  
-                        reader._read_more()
  298
+                        reader.fill()
284 299
                     except EOFError:
285 300
                         buffer.flip()
286 301
                         break
44  lib/concurrence/xmpp/stream.py
@@ -13,86 +13,86 @@ class XMPPStream:
13 13
     MINOR_VERSION = 0
14 14
     DEFAULT_LANGUAGE = "en"
15 15
     NS_XMPP_STREAM_STREAM = "http://etherx.jabber.org/streams"
16  
-    
  16
+
17 17
     NS_JABBER_CLIENT = "jabber:client"
18  
-    
  18
+
19 19
     def __init__(self, stream, stream_uri = NS_XMPP_STREAM_STREAM, default_ns = NS_JABBER_CLIENT):
20 20
         self.stream = BufferedStream(stream)
21 21
         self.stream_uri = stream_uri
22 22
         self.default_ns = default_ns
23 23
         self.reset()
24  
-        
  24
+
25 25
     def reset(self):
26 26
         self.parser = None
27 27
         self.root = None
28  
-        
  28
+
29 29
     def write_bytes(self, s):
30 30
         self.stream.writer.clear()
31 31
         self.stream.writer.write_bytes(s)
32 32
         self.stream.writer.flush()
33  
-        
  33
+
34 34
     def write_start(self, _id, lang = DEFAULT_LANGUAGE, major_version = MAJOR_VERSION, minor_version = MINOR_VERSION, _to = "", _from = "", include_xml_pi = True):
35  
-        if include_xml_pi:         
  35
+        if include_xml_pi:
36 36
             start = """<?xml version='1.0' encoding='utf-8'?>"""
37 37
         else:
38 38
             start = ""
39 39
 
40  
-        start += """<stream:stream xmlns:stream="%s" xmlns="%s" id="%s" xml:lang="%s" version="%d.%d" """ 
  40
+        start += """<stream:stream xmlns:stream="%s" xmlns="%s" id="%s" xml:lang="%s" version="%d.%d" """
41 41
         start = start % (self.stream_uri, self.default_ns, _id, lang, major_version, minor_version)
42  
-    
  42
+
43 43
         if _from: start += ' from="%s"' % _from
44 44
         if _to: start += ' to="%s"' % _to
45  
-    
  45
+
46 46
         start += ">"
47  
- 
  47
+
48 48
         self.write_bytes(start)
49  
-        
  49
+
50 50
     def write_end(self):
51 51
         self.write_bytes("</stream:stream>")
52  
-                
  52
+
53 53
     def write_auth(self, mechanism = 'DIGEST-MD5'):
54 54
         self.write_bytes("""<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='%s'/>""" % mechanism)
55  
-    
  55
+
56 56
     def write_sasl_response(self, response = ''):
57  
-        if response:        
  57
+        if response:
58 58
             self.write_bytes("""<response xmlns='urn:ietf:params:xml:ns:xmpp-sasl'>%s</response>""" % response)
59 59
         else:
60 60
             self.write_bytes("""<response xmlns='urn:ietf:params:xml:ns:xmpp-sasl'/>""")
61  
-    
  61
+
62 62
     def write_bind_request(self, _id, resource):
63 63
         self.write_bytes("""<iq id='%s' type='set'>
64 64
           <bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'>
65 65
             <resource>%s</resource>
66 66
           </bind>
67 67
         </iq>""" % (_id, resource))
68  
-        
  68
+
69 69
     def write_session_request(self, domain, _id):
70 70
         self.write_bytes("""
71 71
         <iq id='%s' type='set' to='%s'>
72 72
           <session xmlns='urn:ietf:params:xml:ns:xmpp-session'/>
73 73
         </iq>""" % (_id, domain))
74  
-        
  74
+
75 75
     def write_presence(self, priority):
76 76
         self.write_bytes("""<presence><priority>%d</priority></presence>""" % priority)
77 77
 
78 78
     def write_message(self, to_jid, msg):
79 79
         self.write_bytes(str("<message to='%s' type='chat'><body>%s</body></message>" % (to_jid, msg)))
80  
-    
  80
+
81 81
     def elements(self):
82  
-        
  82
+
83 83
         if not self.parser:
84 84
             reader = self.stream.reader
85 85
             class f(object):
86 86
                 def read(self, n):
87 87
                     if reader.buffer.remaining == 0:
88 88
                         #read more data into buffer
89  
-                        reader._read_more()
  89
+                        reader.fill()
90 90
                     return reader.buffer.read_bytes(min(n, reader.buffer.remaining))
91 91
 
92 92
             self.parser = iter(iterparse(f(), events=("start", "end")))
93 93
             event, self.root = self.parser.next()
94 94
             level = 0
95  
-        
  95
+
96 96
         for event, element in self.parser:
97 97
             if event == 'start':
98 98
                 level += 1
@@ -103,4 +103,4 @@ def read(self, n):
103 103
                 #TODO clear root