From fe4b1896e77f48760c71e7d6ffcf1a57de2579e2 Mon Sep 17 00:00:00 2001 From: AlissaSquared Date: Tue, 9 Feb 2016 20:43:53 -0600 Subject: [PATCH 01/34] http/server: Optionally use path instead of host/port combination --- http/server.lua | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/http/server.lua b/http/server.lua index abaa9f7f..dfb0f36c 100644 --- a/http/server.lua +++ b/http/server.lua @@ -161,8 +161,9 @@ local server_mt = { --[[ Starts listening on the given socket Takes a table of options: - - `.host`: address to bind to (required) + - `.host`: address to bind to (required if not `.path`) - `.port`: port to bind to (optional if tls isn't `nil`, in which case defaults to 80 for `.tls == false` or 443 if `.tls == true`) + - `.path`: path to a UNIX socket to bind to (required if not `.host`) - `.v6only`: allow ipv6 only (no ipv4-mapped-ipv6) - `.reuseaddr`: turn on SO_REUSEADDR flag? - `.reuseport`: turn on SO_REUSEPORT flag? @@ -176,9 +177,11 @@ Takes a table of options: ]] local function listen(tbl) local tls = tbl.tls - local host = assert(tbl.host, "need host") + local host = tbl.host + local path = tbl.path + assert(host or path, "need host or path") local port = tbl.port - if port == nil then + if port == nil and tbl.host then if tls == true then port = "443" elseif tls == false then @@ -189,11 +192,12 @@ local function listen(tbl) end local ctx = tbl.ctx if ctx == nil and tls ~= false then - ctx = new_ctx(host) + ctx = new_ctx(host or path) end local s = assert(cs.listen{ host = host; port = port; + path = path; v6only = tbl.v6only; reuseaddr = tbl.reuseaddr; reuseport = tbl.reuseport; From 067a641c12ea73ef35090e6aae9b5782a3c8adcb Mon Sep 17 00:00:00 2001 From: AlissaSquared Date: Tue, 9 Feb 2016 21:53:46 -0600 Subject: [PATCH 02/34] spec/server|http/client: UNIX domain sockets for client, and tests --- http/client.lua | 1 + spec/server_spec.lua | 67 +++++++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/http/client.lua b/http/client.lua index 7cda2582..94f5c610 100644 --- a/http/client.lua +++ b/http/client.lua @@ -32,6 +32,7 @@ local function connect(options, timeout) family = options.family; host = options.host; port = options.port; + path = options.path; sendname = options.sendname; v6only = options.v6only; nodelay = true; diff --git a/spec/server_spec.lua b/spec/server_spec.lua index ae50ad19..5603a27f 100644 --- a/spec/server_spec.lua +++ b/spec/server_spec.lua @@ -1,5 +1,5 @@ local TEST_TIMEOUT = 2 -describe("http.server module", function() +describe("http.server module using hostnames", function() local server = require "http.server" local client = require "http.client" local new_headers = require "http.headers".new @@ -13,30 +13,46 @@ describe("http.server module", function() error(err, 2) end end - local function simple_test(tls, version) + local socket_paths = {} + local function simple_test(tls, version, path) local cq = cqueues.new() - local s = server.listen { - host = "localhost"; - port = 0; - } + local options = {} + if path then + socket_paths[#socket_paths + 1] = path + options.path = path + else + options.host = "localhost" + options.port = 0 + end + local s = server.listen(options) assert(s:listen()) - local _, host, port = s:localname() + local host, port + if not path then + local _ + _, host, port = s:localname() + end local on_stream = spy.new(function(stream) stream:get_headers() stream:shutdown() s:shutdown() + print 'shutting down server // spy' end) cq:wrap(function() s:run(on_stream) s:close() + print 'closed // server' end) cq:wrap(function() - local conn = client.connect { - host = host; - port = port; - tls = tls; - version = version; - } + local options = {} + if path then + options.path = path + else + options.host = host + options.port = port + end + options.tls = tls + options.version = version + local conn = client.connect(options) local stream = conn:new_stream() local headers = new_headers() headers:append(":method", "GET") @@ -45,21 +61,38 @@ describe("http.server module", function() assert(stream:write_headers(headers, true)) stream:get_headers() conn:close() + print 'closed connection // client' end) assert_loop(cq, TEST_TIMEOUT) assert.truthy(cq:empty()) assert.spy(on_stream).was.called() end - it("works with plain http 1.1", function() + it("works with plain http 1.1 using IP", function() simple_test(false, 1.1) end) - it("works with https 1.1", function() + it("works with https 1.1 using IP", function() simple_test(true, 1.1) end) - it("works with plain http 2.0", function() + it("works with plain http 2.0 using IP", function() simple_test(false, 2.0) end); - (require "http.tls".has_alpn and it or pending)("works with https 2.0", function() + (require "http.tls".has_alpn and it or pending)("works with https 2.0 using IP", function() simple_test(true, 2.0) end) + local socket_path = os.tmpname() .. ".sock" + it("works with plain http 1.1 using UNIX socket domain", function() + simple_test(false, 1.1, os.tmpname() .. ".socket") + end) + it("works with https 1.1 using UNIX socket domain", function() + simple_test(true, 1.1, os.tmpname() .. ".socket") + end) + it("works with plain http 2.0 using UNIX socket domain", function() + simple_test(false, 2.0, os.tmpname() .. ".socket") + end); + (require "http.tls".has_alpn and it or pending)("works with https 2.0 using UNIX socket domain", function() + simple_test(true, 2.0, os.tmpname() .. ".socket") + end) + for k, v in pairs(socket_paths) do + os.remove(v) + end end) From 1c8612c7144e3dbf6fc8200d3af2c593b0d5df7a Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 11 Feb 2016 14:16:31 +1100 Subject: [PATCH 03/34] LICENSE: Update copyright year --- LICENSE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.md b/LICENSE.md index a467327b..0adb5f6e 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 Daurnimator +Copyright (c) 2015-2016 Daurnimator Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From d92207a5f6227cc0b3b6cf3f46a98412b0aca6ca Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 12 Feb 2016 15:48:47 +1100 Subject: [PATCH 04/34] doc/index.md: Use plural --- doc/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/index.md b/doc/index.md index e589ee7b..002daaa1 100644 --- a/doc/index.md +++ b/doc/index.md @@ -664,13 +664,13 @@ Set to `math.huge` to not give up. ### `request.post301` {#http.request.post301} -Respect RFC 2616 Section 10.3.2 and **don't** convert POST requests into body-less GET requests when following a 301 redirect. The non-RFC behaviour is ubiquitous in web browsers and assumed by server. Modern HTTP endpoints send status code 308 to indicate that they don't want the method to be changed. +Respect RFC 2616 Section 10.3.2 and **don't** convert POST requests into body-less GET requests when following a 301 redirect. The non-RFC behaviour is ubiquitous in web browsers and assumed by servers. Modern HTTP endpoints send status code 308 to indicate that they don't want the method to be changed. Defaults to `false`. ### `request.post302` {#http.request.post302} -Respect RFC 2616 Section 10.3.3 and **don't** convert POST requests into body-less GET requests when following a 302 redirect. The non-RFC behaviour is ubiquitous in web browsers and assumed by server. Modern HTTP endpoints send status code 307 to indicate that they don't want the method to be changed. +Respect RFC 2616 Section 10.3.3 and **don't** convert POST requests into body-less GET requests when following a 302 redirect. The non-RFC behaviour is ubiquitous in web browsers and assumed by servers. Modern HTTP endpoints send status code 307 to indicate that they don't want the method to be changed. Defaults to `false`. From f931c2a8a12bbf3ba9a35c7b2b3f20bb3bdc25de Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 12 Feb 2016 15:55:11 +1100 Subject: [PATCH 05/34] http/server: Keep a condition variable to be alerted of shutdown. Fixes #9 --- http/server.lua | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/http/server.lua b/http/server.lua index dfb0f36c..bb7cc67a 100644 --- a/http/server.lua +++ b/http/server.lua @@ -213,6 +213,7 @@ local function listen(tbl) ctx = ctx; max_concurrent = tbl.max_concurrent; n_connections = 0; + shutdown_cond = cc.new(); connection_done = cc.new(); -- signalled when connection has been closed client_timeout = tbl.client_timeout; }, server_mt) @@ -228,8 +229,13 @@ function server_methods:localname() return self.socket:localname() end +-- Shutdown the server socket +-- does *not* shutdown connections that originated on this socket function server_methods:shutdown() - self.socket:shutdown() + if self.shutdown_cond then + self.shutdown_cond:signal() + end + self.shutdown_cond = nil end function server_methods:close() @@ -239,19 +245,20 @@ function server_methods:close() self.socket:close() end --- accepts a new client and returns it as an http connection object function server_methods:run(on_stream, cq) cq = cq or cqueues.running() - while true do + while self.shutdown_cond do if self.n_connections >= self.max_concurrent then self.connection_done:wait() end -- Yield this thread until a client arrives - local socket, accept_err = self.socket:accept{nodelay = true;} + local socket, accept_err = self.socket:accept({nodelay = true;}, 0) if socket == nil then - if accept_err == ce.EINVAL then - -- has been shutdown - break + if accept_err == ce.ETIMEDOUT then + -- Wait for a client + if cqueues.poll(self.socket, self.shutdown_cond) == self.shutdown_cond then + break + end elseif accept_err == ce.EMFILE then -- Wait for another request to finish if not self.connection_done:wait(0.1) then From 18cac34800e8851237d313af11d7b40a169fbb80 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 12 Feb 2016 17:23:07 +1100 Subject: [PATCH 06/34] doc/index: Document current http.server methods --- doc/index.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/index.md b/doc/index.md index 002daaa1..9c3dbd1d 100644 --- a/doc/index.md +++ b/doc/index.md @@ -693,7 +693,26 @@ On success, returns the response [*headers*](#http.headers) and a [*stream*](#st ## http.server -### `listen(options)` {#http.client.connect} +This interface is **unstable**. + +### `listen(options)` {#http.server.connect} + + +### `server:listen(timeout)` {#http.server:listen} + + +### `server:localname()` {#http.server:localname} + + +### `server:pause()` {#http.server:pause} + +Cause [`server:run`](#http.server:run) to stop processing new clients and return. + + +### `server:close()` {#http.server:close} + + +### `server:run(on_stream, cq)` {#http.server:run} ## http.stream_common From 0e959e4a1e787dea6fbf689548583732ce0aad68 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Fri, 12 Feb 2016 18:15:30 +1100 Subject: [PATCH 07/34] README: Add luarocks install instuctions Remember to remove the --server option once released --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index a8116ef1..926599e7 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,11 @@ This project is a work in progress and not ready for production use. # Installation +It's recommended to install lua-http by using [luarocks](https://luarocks.org/). +This will automatically install run-time lua dependencies for you. + + $ luarocks install --server=http://luarocks.org/dev http + ## Dependencies - [cqueues](http://25thandclement.com/~william/projects/cqueues.html) >= 20150907 From 1311c35b840bb9bde09bbb1900b46c7b166b38a8 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 15:51:03 +1100 Subject: [PATCH 08/34] spec/request_spec: Put handle_redirect tests into separate tests --- spec/request_spec.lua | 148 +++++++++++++++++++++--------------------- 1 file changed, 73 insertions(+), 75 deletions(-) diff --git a/spec/request_spec.lua b/spec/request_spec.lua index 657fb0b3..551ff5f6 100644 --- a/spec/request_spec.lua +++ b/spec/request_spec.lua @@ -119,81 +119,79 @@ describe("http.request module", function() req:set_body(io.tmpfile()) assert.same("100-continue", req.headers:get("expect")) end) + local headers = require "http.headers" it(":handle_redirect works", function() - local headers = require "http.headers" - do - local orig_req = request.new_from_uri("http://example.com") - local orig_headers = headers.new() - orig_headers:append(":status", "301") - orig_headers:append("location", "/foo") - local new_req = orig_req:handle_redirect(orig_headers) - -- same - assert.same(orig_req.host, new_req.host) - assert.same(orig_req.port, new_req.port) - assert.same(orig_req.tls, new_req.tls) - assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") - assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") - assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") - assert.same(orig_req.body, new_req.body) - -- different - assert.same("/foo", new_req.headers:get ":path") - assert.same(orig_req.max_redirects-1, new_req.max_redirects) - end - do - local orig_req = request.new_from_uri("http://example.com") - local orig_headers = headers.new() - orig_headers:append(":status", "302") - orig_headers:append("location", "//blah.com:1234/example") - local new_req = orig_req:handle_redirect(orig_headers) - -- same - assert.same(orig_req.tls, new_req.tls) - assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") - assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") - assert.same(orig_req.body, new_req.body) - -- different - assert.same("blah.com", new_req.host) - assert.same(1234, new_req.port) - assert.same("blah.com:1234", new_req.headers:get ":authority") - assert.same("/example", new_req.headers:get ":path") - assert.same(orig_req.max_redirects-1, new_req.max_redirects) - end - do -- maximum redirects exceeded - local ce = require "cqueues.errno" - local orig_req = request.new_from_uri("http://example.com") - orig_req.max_redirects = 0 - local orig_headers = headers.new() - orig_headers:append(":status", "302") - orig_headers:append("location", "/") - assert.same({nil, "maximum redirects exceeded", ce.ELOOP}, {orig_req:handle_redirect(orig_headers)}) - end - do -- missing location header - local ce = require "cqueues.errno" - local orig_req = request.new_from_uri("http://example.com") - local orig_headers = headers.new() - orig_headers:append(":status", "302") - assert.same({nil, "missing location header for redirect", ce.EINVAL}, {orig_req:handle_redirect(orig_headers)}) - end - do -- POST => GET transformation - local orig_req = request.new_from_uri("http://example.com") - orig_req.headers:upsert(":method", "POST") - orig_req.headers:upsert("content-type", "text/plain") - orig_req:set_body("foo") - local orig_headers = headers.new() - orig_headers:append(":status", "303") - orig_headers:append("location", "/foo") - local new_req = orig_req:handle_redirect(orig_headers) - -- same - assert.same(orig_req.host, new_req.host) - assert.same(orig_req.port, new_req.port) - assert.same(orig_req.tls, new_req.tls) - assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") - assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") - -- different - assert.same("GET", new_req.headers:get ":method") - assert.same("/foo", new_req.headers:get ":path") - assert.falsy(new_req.headers:has "content-type") - assert.same(nil, new_req.body) - assert.same(orig_req.max_redirects-1, new_req.max_redirects) - end + local orig_req = request.new_from_uri("http://example.com") + local orig_headers = headers.new() + orig_headers:append(":status", "301") + orig_headers:append("location", "/foo") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.host, new_req.host) + assert.same(orig_req.port, new_req.port) + assert.same(orig_req.tls, new_req.tls) + assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") + assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") + assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") + assert.same(orig_req.body, new_req.body) + -- different + assert.same("/foo", new_req.headers:get ":path") + assert.same(orig_req.max_redirects-1, new_req.max_redirects) + end) + it(":handle_redirect works", function() + local orig_req = request.new_from_uri("http://example.com") + local orig_headers = headers.new() + orig_headers:append(":status", "302") + orig_headers:append("location", "//blah.com:1234/example") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.tls, new_req.tls) + assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") + assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") + assert.same(orig_req.body, new_req.body) + -- different + assert.same("blah.com", new_req.host) + assert.same(1234, new_req.port) + assert.same("blah.com:1234", new_req.headers:get ":authority") + assert.same("/example", new_req.headers:get ":path") + assert.same(orig_req.max_redirects-1, new_req.max_redirects) + end) + it(":handle_redirect detects maximum redirects exceeded", function() + local ce = require "cqueues.errno" + local orig_req = request.new_from_uri("http://example.com") + orig_req.max_redirects = 0 + local orig_headers = headers.new() + orig_headers:append(":status", "302") + orig_headers:append("location", "/") + assert.same({nil, "maximum redirects exceeded", ce.ELOOP}, {orig_req:handle_redirect(orig_headers)}) + end) + it(":handle_redirect detects missing location header", function() + local ce = require "cqueues.errno" + local orig_req = request.new_from_uri("http://example.com") + local orig_headers = headers.new() + orig_headers:append(":status", "302") + assert.same({nil, "missing location header for redirect", ce.EINVAL}, {orig_req:handle_redirect(orig_headers)}) + end) + it(":handle_redirect detects POST => GET transformation", function() + local orig_req = request.new_from_uri("http://example.com") + orig_req.headers:upsert(":method", "POST") + orig_req.headers:upsert("content-type", "text/plain") + orig_req:set_body("foo") + local orig_headers = headers.new() + orig_headers:append(":status", "303") + orig_headers:append("location", "/foo") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.host, new_req.host) + assert.same(orig_req.port, new_req.port) + assert.same(orig_req.tls, new_req.tls) + assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") + assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") + -- different + assert.same("GET", new_req.headers:get ":method") + assert.same("/foo", new_req.headers:get ":path") + assert.falsy(new_req.headers:has "content-type") + assert.same(nil, new_req.body) + assert.same(orig_req.max_redirects-1, new_req.max_redirects) end) end) From fe62e33c52dc5f77d893599b3a51fa75dee8d8e9 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 15:52:06 +1100 Subject: [PATCH 09/34] http/request: Add :clone method --- doc/index.md | 9 +++++++++ http/request.lua | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/doc/index.md b/doc/index.md index 9c3dbd1d..b9491d0c 100644 --- a/doc/index.md +++ b/doc/index.md @@ -683,6 +683,15 @@ Allows setting a request body. `body` may be a string, function or lua file obje - If `body` is a lua file object, it will be [`:seek`'d](http://www.lua.org/manual/5.3/manual.html#pdf-file:seek) to the start, then sent as a body. Any errors encountered during file operations **will be thrown**. +### `request:clone()` {#http.request:clone} + +Creates and returns a clone of the request. + +The clone has its own deep copy of the [`.headers`](#http.request.headers) field. + +The [`.tls`](#http.request.tls) and body fields are shallow copied from the original request. + + ### `request:go(timeout)` {#http.request:timeout} Performs the request. diff --git a/http/request.lua b/http/request.lua index 0a8d013a..3891b2a2 100644 --- a/http/request.lua +++ b/http/request.lua @@ -117,6 +117,25 @@ local function new_from_stream(stream) return self end +function request_methods:clone() + return setmetatable({ + host = self.host; + port = self.port; + tls = self.tls; + sendname = self.sendname; + version = self.version; + + headers = self.headers:clone(); + body = self.body; + + expect_100_timeout = rawget(self, "expect_100_timeout"); + follow_redirects = rawget(self, "follow_redirects"); + max_redirects = rawget(self, "max_redirects"); + post301 = rawget(self, "post301"); + post302 = rawget(self, "post302"); + }, request_mt) +end + function request_methods:to_url() -- TODO: userinfo section (username/password) local method = self.headers:get(":method") From 9d0892b1893c801296f58a56f2dda9e56f56a457 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 16:22:26 +1100 Subject: [PATCH 10/34] http/request: Implement :handle_redirect by :clone-ing then changing fields Fixes issue where a custom host/port were not retained over a relative redirect --- http/request.lua | 88 ++++++++++++++++++++++++++++--------------- spec/request_spec.lua | 21 +++++++++++ 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/http/request.lua b/http/request.lua index 3891b2a2..c5f646c2 100644 --- a/http/request.lua +++ b/http/request.lua @@ -179,37 +179,63 @@ function request_methods:handle_redirect(orig_headers) return nil, "missing location header for redirect", ce.EINVAL end local uri_t = assert(uri_ref:match(location), "invalid URI") - local orig_scheme = self.headers:get(":scheme") - if uri_t.scheme == nil then - uri_t.scheme = orig_scheme + local new_req = self:clone() + new_req.max_redirects = max_redirects - 1 + local is_connect = new_req.headers:get(":method") == "CONNECT" + if uri_t.scheme ~= nil then + if not is_connect then + new_req.headers:upsert(":scheme", uri_t.scheme) + end + if uri_t.scheme == "https" or uri_t.scheme == "wss" then + new_req.tls = self.tls or true + else + new_req.tls = false + end + end + if uri_t.host ~= nil then + new_req.host, new_req.port = uri_t.host, uri_t.port + if not is_connect then + new_req.headers:upsert(":authority", http_util.to_authority(uri_t.host, uri_t.port, new_req.headers:get(":scheme"))) + end + new_req.sendname = nil end - if uri_t.host == nil then - uri_t.host, uri_t.port = http_util.split_authority(self.headers:get(":authority"), orig_scheme) + if is_connect then + assert(uri_t.path == "", "CONNECT requests cannot have a path") + assert(uri_t.query == nil, "CONNECT requests cannot have a query") + else + local new_path + if uri_t.path == "" then + new_path = "/" + else + new_path = http_util.encodeURI(uri_t.path) + if new_path:sub(1, 1) ~= "/" then -- relative path + local orig_target = self.headers:get(":path") + local orig_path = assert(uri_ref:match(orig_target)).path + orig_path = http_util.encodeURI(orig_path) + new_path = http_util.resolve_relative_path(orig_path, new_path) + end + end + if uri_t.query then + new_path = new_path .. "?" .. http_util.encodeURI(uri_t.query) + end + new_req.headers:upsert(":path", new_path) end - if uri_t.path ~= nil then - uri_t.path = http_util.encodeURI(uri_t.path) - if uri_t.path:sub(1, 1) ~= "/" then -- relative path - local orig_target = self.headers:get(":path") - local orig_path = assert(uri_ref:match(orig_target)).path - orig_path = http_util.encodeURI(orig_path) - uri_t.path = http_util.resolve_relative_path(orig_path, uri_t.path) + if uri_t.userinfo then + local field + if is_connect then + field = "proxy-authorization" + else + field = "authorization" end + new_req.headers:upsert(field, "basic " .. basexx.to_base64(uri_t.userinfo), true) end - local headers = self.headers:clone() - local new_req = new_from_uri_t(uri_t, headers) - new_req.expect_100_timeout = rawget(self, "expect_100_timeout") - new_req.follow_redirects = rawget(self, "follow_redirects") - new_req.max_redirects = max_redirects - 1 - new_req.post301 = rawget(self, "post301") - new_req.post302 = rawget(self, "post302") if not new_req.tls and self.tls then --[[ RFC 7231 5.5.2: A user agent MUST NOT send a Referer header field in an unsecured HTTP request if the referring page was received with a secure protocol.]] - headers:delete("referer") + new_req.headers:delete("referer") else - headers:upsert("referer", self:to_url()) + new_req.headers:upsert("referer", self:to_url()) end - new_req.body = self.body -- Change POST requests to a body-less GET on redirect? local orig_status = orig_headers:get(":status") if (orig_status == "303" @@ -217,19 +243,19 @@ function request_methods:handle_redirect(orig_headers) or (orig_status == "302" and not self.post302) ) and self.headers:get(":method") == "POST" then - headers:upsert(":method", "GET") + new_req.headers:upsert(":method", "GET") -- Remove headers that don't make sense without a body -- Headers that require a body - headers:delete("transfer-encoding") - headers:delete("content-length") + new_req.headers:delete("transfer-encoding") + new_req.headers:delete("content-length") -- Representation Metadata from RFC 7231 Section 3.1 - headers:delete("content-encoding") - headers:delete("content-language") - headers:delete("content-location") - headers:delete("content-type") + new_req.headers:delete("content-encoding") + new_req.headers:delete("content-language") + new_req.headers:delete("content-location") + new_req.headers:delete("content-type") -- Other... - if headers:get("expect") == "100-continue" then - headers:delete("expect") + if new_req.headers:get("expect") == "100-continue" then + new_req.headers:delete("expect") end new_req.body = nil end diff --git a/spec/request_spec.lua b/spec/request_spec.lua index 551ff5f6..70ed75a4 100644 --- a/spec/request_spec.lua +++ b/spec/request_spec.lua @@ -194,4 +194,25 @@ describe("http.request module", function() assert.same(nil, new_req.body) assert.same(orig_req.max_redirects-1, new_req.max_redirects) end) + it(":handle_redirect deletes keeps original custom host, port and sendname if relative", function() + local orig_req = request.new_from_uri("http://example.com") + orig_req.host = "other.com" + orig_req.sendname = "something.else" + local orig_headers = headers.new() + orig_headers:append(":status", "301") + orig_headers:append("location", "/foo") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.host, new_req.host) + assert.same(orig_req.port, new_req.port) + assert.same(orig_req.tls, new_req.tls) + assert.same(orig_req.sendname, new_req.sendname) + assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") + assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") + assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") + assert.same(orig_req.body, new_req.body) + -- different + assert.same("/foo", new_req.headers:get ":path") + assert.same(orig_req.max_redirects-1, new_req.max_redirects) + end) end) From c89b285d67ccec04a46fd002a97ed271f630683c Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 17:07:33 +1100 Subject: [PATCH 11/34] spec/util_spec: Add tests for split_authority --- spec/util_spec.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spec/util_spec.lua b/spec/util_spec.lua index 74deb6ef..1584db10 100644 --- a/spec/util_spec.lua +++ b/spec/util_spec.lua @@ -53,6 +53,16 @@ describe("http.util module", function() assert.same(t, r) end end) + it("split_authority works", function() + assert.same({"example.com", 80}, {util.split_authority("example.com", "http")}) + assert.same({"example.com", 8000}, {util.split_authority("example.com:8000", "http")}) + assert.has.errors(function() + util.split_authority("example.com", "madeupscheme") + end) + -- IPv6 + assert.same({"::1", 443}, {util.split_authority("[::1]", "https")}) + assert.same({"::1", 8000}, {util.split_authority("[::1]:8000", "https")}) + end) it("split_header works correctly", function() -- nil assert.same({n=0}, util.split_header(nil)) From 01225a890926cb4e31526c078970ea6dca8ce1ab Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 17:11:01 +1100 Subject: [PATCH 12/34] spec/util_spec: Add tests for to_authority --- spec/util_spec.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spec/util_spec.lua b/spec/util_spec.lua index 1584db10..0c0cd4f4 100644 --- a/spec/util_spec.lua +++ b/spec/util_spec.lua @@ -63,6 +63,13 @@ describe("http.util module", function() assert.same({"::1", 443}, {util.split_authority("[::1]", "https")}) assert.same({"::1", 8000}, {util.split_authority("[::1]:8000", "https")}) end) + it("to_authority works", function() + assert.same("example.com", util.to_authority("example.com", 80, "http")) + assert.same("example.com:8000", util.to_authority("example.com", 8000, "http")) + -- IPv6 + assert.same("[::1]", util.to_authority("::1", 443, "https")) + assert.same("[::1]:8000", util.to_authority("::1", 8000, "https")) + end) it("split_header works correctly", function() -- nil assert.same({n=0}, util.split_header(nil)) From e42b15c2cff6a911a41d40ab914d1af7d638b85e Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 17:15:17 +1100 Subject: [PATCH 13/34] spec/util_spec: Add test for decodeURI --- spec/util_spec.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/util_spec.lua b/spec/util_spec.lua index 0c0cd4f4..457965f6 100644 --- a/spec/util_spec.lua +++ b/spec/util_spec.lua @@ -1,6 +1,9 @@ describe("http.util module", function() local unpack = table.unpack or unpack -- luacheck: ignore 113 local util = require "http.util" + it("decodeURI works", function() + assert.same("Encoded string", util.decodeURI("Encoded%20string")) + end) it("decodeURI doesn't decode blacklisted characters", function() assert.same("%24", util.decodeURI("%24")) local s = util.encodeURIComponent("#$&+,/:;=?@") From c6422f6072210dd0d0c1baa23e2e69aa100be99d Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 18:07:03 +1100 Subject: [PATCH 14/34] README: Remove TODO section, add Features, update Status --- README.md | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 926599e7..1b6569a8 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ ## Features - Optionally asynchronous (including DNS lookups and SSL) + - Supports HTTP version 1.0, 1.1 and 2 + - Functionality for both client and server + - Websockets - Compatible with Lua 5.1, 5.2, 5.3 and [LuaJIT](http://luajit.org/) @@ -11,25 +14,13 @@ Can be found at [https://daurnimator.github.io/lua-http/](https://daurnimator.github.io/lua-http/) -# Status - -This project is a work in progress and not ready for production use. +## Status [![Build Status](https://travis-ci.org/daurnimator/lua-http.svg)](https://travis-ci.org/daurnimator/lua-http) [![Coverage Status](https://coveralls.io/repos/daurnimator/lua-http/badge.svg?branch=master&service=github)](https://coveralls.io/github/daurnimator/lua-http?branch=master) -## Todo - - - [x] HTTP 1.1 - - [x] [HTTP 2](https://http2.github.io/http2-spec/) - - [x] [HPACK](https://http2.github.io/http2-spec/compression.html) - - [ ] Connection pooling - - [ ] [`socket.http`](http://w3.impa.br/~diego/software/luasocket/http.html) compatibility layer - - [x] Prosody [`net.http`](https://prosody.im/doc/developers/net/http) compatibility layer - - [x] Handle redirects - - [ ] Be able to use an HTTP proxy - - [x] Compression (e.g. gzip) - - [ ] Websockets + - HTTP client API is reaching stability + - The HTTP server API is still changing # Installation @@ -47,7 +38,7 @@ This will automatically install run-time lua dependencies for you. - [lpeg_patterns](https://github.com/daurnimator/lpeg_patterns) >= 0.2 - [fifo](https://github.com/daurnimator/fifo.lua) -If you want to use gzip compression you will need **one** of: +To use gzip compression you need **one** of: - [lzlib](https://github.com/LuaDist/lzlib) or [lua-zlib](https://github.com/brimworks/lua-zlib) From 823a7cee6407fa8352879defe0a6e7947ace69ed Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 18:20:04 +1100 Subject: [PATCH 15/34] http/h2_{connection,stream}: Don't rely on compat53 when in lua 5.3 --- http/h2_connection.lua | 6 +++++- http/h2_stream.lua | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/http/h2_connection.lua b/http/h2_connection.lua index d20d9a57..4a0e8a21 100644 --- a/http/h2_connection.lua +++ b/http/h2_connection.lua @@ -9,10 +9,14 @@ local h2_error = require "http.h2_error" local h2_stream = require "http.h2_stream" local hpack = require "http.hpack" local h2_banned_ciphers = require "http.tls".banned_ciphers -local assert = require "compat53.module".assert local spack = string.pack or require "compat53.string".pack local sunpack = string.unpack or require "compat53.string".unpack +local assert = assert +if _VERSION:match("%d+%.?%d*") < "5.3" then + assert = require "compat53.module".assert +end + local function xor(a, b) return (a and b) or not (a or b) end diff --git a/http/h2_stream.lua b/http/h2_stream.lua index 66ffdcda..d74911ff 100644 --- a/http/h2_stream.lua +++ b/http/h2_stream.lua @@ -7,11 +7,15 @@ local band = require "http.bit".band local bor = require "http.bit".bor local h2_errors = require "http.h2_error".errors local stream_common = require "http.stream_common" -local assert = require "compat53.module".assert local spack = string.pack or require "compat53.string".pack local sunpack = string.unpack or require "compat53.string".unpack local unpack = table.unpack or unpack -- luacheck: ignore 113 +local assert = assert +if _VERSION:match("%d+%.?%d*") < "5.3" then + assert = require "compat53.module".assert +end + local function xor(a, b) return (a and b) or not (a or b) end From a73acef96545ab366522ff3d2a8ec436f6724961 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 15 Feb 2016 19:35:15 +1100 Subject: [PATCH 16/34] http/h1_stream: On close_when_done, remove self from pipeline *before* shutting down, but signal other streams *after* --- http/h1_stream.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/http/h1_stream.lua b/http/h1_stream.lua index 8d648a82..d6200414 100644 --- a/http/h1_stream.lua +++ b/http/h1_stream.lua @@ -99,11 +99,11 @@ function stream_methods:set_state(new) -- If we have just finished writing the response if (old == "idle" or old == "open" or old == "half closed (remote)") and (new == "half closed (local)" or new == "closed") then + -- remove ourselves from the write pipeline + assert(self.connection.pipeline:pop() == self) if self.close_when_done then self.connection:shutdown() end - -- remove ourselves from the write pipeline - assert(self.connection.pipeline:pop() == self) local next_stream = self.connection.pipeline:peek() if next_stream then next_stream.pipeline_cond:signal() @@ -124,11 +124,11 @@ function stream_methods:set_state(new) -- If we have just finished reading the response; if (old == "idle" or old == "open" or old == "half closed (local)") and (new == "half closed (remote)" or new == "closed") then + -- remove ourselves from the read pipeline + assert(self.connection.pipeline:pop() == self) if self.close_when_done then self.connection:shutdown() end - -- remove ourselves from the read pipeline - assert(self.connection.pipeline:pop() == self) local next_stream = self.connection.pipeline:peek() if next_stream then next_stream.pipeline_cond:signal() From b62814d0e1960ed98627985100f8a1f657b2183d Mon Sep 17 00:00:00 2001 From: daurnimator Date: Tue, 16 Feb 2016 11:36:03 +1100 Subject: [PATCH 17/34] http/h1_connection: get_next_incoming_stream now blocks until there is data available --- http/h1_connection.lua | 29 ++++++++++++++++------------- spec/h1_connection_spec.lua | 12 +----------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/http/h1_connection.lua b/http/h1_connection.lua index 2fcf7ce7..0f60ea29 100644 --- a/http/h1_connection.lua +++ b/http/h1_connection.lua @@ -120,22 +120,25 @@ end -- this function *should never throw* function connection_methods:get_next_incoming_stream(timeout) assert(self.type == "server") + local deadline = timeout and (monotime()+timeout) -- Make sure we don't try and read before the previous request has been fully read - if self.req_locked then + repeat -- Wait until previous requests have been fully read - if not self.req_cond:wait(timeout) then - return nil, ce.ETIMEDOUT + if self.req_locked then + if not self.req_cond:wait(deadline and deadline - monotime()) then + return nil, ce.ETIMEDOUT + end + assert(self.req_locked == nil) end - assert(self.req_locked == nil) - end - if self.socket == nil or self.socket:eof("r") then - return nil, ce.EPIPE - end - -- check if socket has already got an error set - local errno = self.socket:error("r") - if errno then - return nil, onerror(self.socket, "read", errno, 3) - end + if self.socket == nil then + return nil, ce.EPIPE + end + -- Wait for at least one byte + local ok, err, errno = self.socket:fill(1, deadline and deadline-monotime()) + if not ok then + return nil, err or ce.EPIPE, errno + end + until not self.req_locked local stream = h1_stream.new(self) self.pipeline:push(stream) self.req_locked = stream diff --git a/spec/h1_connection_spec.lua b/spec/h1_connection_spec.lua index 47b22a0c..95c3ddad 100644 --- a/spec/h1_connection_spec.lua +++ b/spec/h1_connection_spec.lua @@ -402,7 +402,6 @@ describe("low level http 1 connection operations", function() end) describe("high level http1 connection operations", function() local h1_connection = require "http.h1_connection" - local cqueues = require "cqueues" local cs = require "cqueues.socket" local ce = require "cqueues.errno" @@ -415,16 +414,7 @@ describe("high level http1 connection operations", function() it(":get_next_incoming_stream times out", function() local s, c = new_pair(1.1) -- luacheck: ignore 211 - local cq = cqueues.new() - cq:wrap(function() - local stream = s:get_next_incoming_stream() - cq:wrap(function() - assert.same({nil, ce.ETIMEDOUT}, {s:get_next_incoming_stream(0.05)}) - end) - cqueues.sleep(0.1) - stream:shutdown() - end) - assert(cq:loop()) + assert.same({nil, ce.ETIMEDOUT}, {s:get_next_incoming_stream(0.05)}) end) it(":get_next_incoming_stream returns nil, EPIPE when no data", function() local s, c = new_pair(1.1) From 3083c2ae6ace2f54d75720a1507850327fa709c8 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Wed, 17 Feb 2016 22:26:44 +1100 Subject: [PATCH 18/34] .travis.yml: Run luacheck --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1245717a..2ca26060 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ before_install: # into 'here' subdirectory - export PATH=$PATH:$PWD/here/bin # Add directory with all installed binaries to PATH - eval `luarocks path --bin` + - luarocks install luacheck - luarocks install luacov-coveralls - luarocks install busted @@ -43,6 +44,7 @@ install: - if [ "$ZLIB" = "lua-zlib" ]; then luarocks install lua-zlib; fi script: + - luacheck . - busted -c after_success: From 415341d7c473f03a225166550a74d266920357ac Mon Sep 17 00:00:00 2001 From: daurnimator Date: Wed, 17 Feb 2016 22:35:53 +1100 Subject: [PATCH 19/34] .travis.yml: Install hererocks out of tree (fixes luacheck failure) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ca26060..94227f5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,10 +29,10 @@ branches: before_install: - pip install hererocks - - hererocks here -r^ --$LUA # Install latest LuaRocks version + - hererocks ~/hererocks -r^ --$LUA # Install latest LuaRocks version # plus the Lua version for this build job # into 'here' subdirectory - - export PATH=$PATH:$PWD/here/bin # Add directory with all installed binaries to PATH + - export PATH=$PATH:~/hererocks/bin # Add directory with all installed binaries to PATH - eval `luarocks path --bin` - luarocks install luacheck - luarocks install luacov-coveralls From 2a123794eca6073007e7a0e121b9a173267af7fa Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 18 Feb 2016 01:06:42 +1100 Subject: [PATCH 20/34] http/request: Update port on scheme change --- http/request.lua | 6 ++++-- spec/request_spec.lua | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/http/request.lua b/http/request.lua index c5f646c2..e200a38b 100644 --- a/http/request.lua +++ b/http/request.lua @@ -193,9 +193,11 @@ function request_methods:handle_redirect(orig_headers) end end if uri_t.host ~= nil then - new_req.host, new_req.port = uri_t.host, uri_t.port + local new_scheme = new_req.headers:get(":scheme") + new_req.host = uri_t.host + new_req.port = uri_t.port or http_util.scheme_to_port[new_scheme] if not is_connect then - new_req.headers:upsert(":authority", http_util.to_authority(uri_t.host, uri_t.port, new_req.headers:get(":scheme"))) + new_req.headers:upsert(":authority", http_util.to_authority(uri_t.host, uri_t.port, new_scheme)) end new_req.sendname = nil end diff --git a/spec/request_spec.lua b/spec/request_spec.lua index 70ed75a4..6b04ed90 100644 --- a/spec/request_spec.lua +++ b/spec/request_spec.lua @@ -138,7 +138,27 @@ describe("http.request module", function() assert.same("/foo", new_req.headers:get ":path") assert.same(orig_req.max_redirects-1, new_req.max_redirects) end) - it(":handle_redirect works", function() + it(":handle_redirect works with cross-scheme port-less uri", function() + local orig_req = request.new_from_uri("http://example.com") + local orig_headers = headers.new() + orig_headers:append(":status", "302") + orig_headers:append("location", "https://blah.com/example") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") + assert.same(orig_req.body, new_req.body) + -- different + assert.same(false, orig_req.tls) + assert.same(true, new_req.tls) + assert.same("https", new_req.headers:get ":scheme") + assert.same("blah.com", new_req.host) + assert.same(80, orig_req.port) + assert.same(443, new_req.port) + assert.same("blah.com", new_req.headers:get ":authority") + assert.same("/example", new_req.headers:get ":path") + assert.same(orig_req.max_redirects-1, new_req.max_redirects) + end) + it(":handle_redirect works with scheme relative uri", function() local orig_req = request.new_from_uri("http://example.com") local orig_headers = headers.new() orig_headers:append(":status", "302") From f7475d0b8166a5975cb16f50489d8579e2791c72 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 18 Feb 2016 01:14:29 +1100 Subject: [PATCH 21/34] .travis.yml: Add test for lua 5.3 without compat53 installed --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 94227f5b..ebd1a338 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ env: - LUA="lua 5.2" ZLIB=lua-zlib - LUA="lua 5.3" - LUA="lua 5.3" ZLIB=lzlib + - LUA="lua 5.3" COMPAT53=no # lua-zlib is currently unavailable for lua5.3 https://github.com/brimworks/lua-zlib/issues/28 - LUA="luajit @" - LUA="luajit @" ZLIB=lzlib @@ -42,6 +43,7 @@ install: - luarocks install --only-deps http-scm-0.rockspec - if [ "$ZLIB" = "lzlib" ]; then luarocks install lzlib; fi - if [ "$ZLIB" = "lua-zlib" ]; then luarocks install lua-zlib; fi + - if [ "$COMPAT53" = "no" ]; then luarocks remove compat53; fi script: - luacheck . From 08b14e78ac403598299a80608d6fc7cde632bfb4 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 18 Feb 2016 01:35:27 +1100 Subject: [PATCH 22/34] http/h1_stream: Don't allow server to send headers while stream is idle At that time: - request method is unknown - support for transfer-encodings is unknown - peer HTTP version is unknown --- http/h1_stream.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/h1_stream.lua b/http/h1_stream.lua index d6200414..7c796b39 100644 --- a/http/h1_stream.lua +++ b/http/h1_stream.lua @@ -370,6 +370,9 @@ function stream_methods:write_headers(headers, end_stream, timeout) end local status_code, method if self.type == "server" then + if self.state == "idle" then + error("cannot write headers when stream is idle") + end -- Make sure we're at the front of the pipeline if self.connection.pipeline:peek() ~= self then if not self.pipeline_cond:wait(deadline and (deadline-monotime)) then From bc20d500ce7818c621c7fb49fe20722ca7386c4b Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 18 Feb 2016 01:37:55 +1100 Subject: [PATCH 23/34] spec/compat_prosody_spec: Add full end-to-end test --- spec/compat_prosody_spec.lua | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/spec/compat_prosody_spec.lua b/spec/compat_prosody_spec.lua index e3bcdefd..eff11593 100644 --- a/spec/compat_prosody_spec.lua +++ b/spec/compat_prosody_spec.lua @@ -1,6 +1,18 @@ +local TEST_TIMEOUT = 2 describe("http.compat.prosody module", function() local cqueues = require "cqueues" local request = require "http.compat.prosody".request + local new_headers = require "http.headers".new + local server = require "http.server" + local function assert_loop(cq, timeout) + local ok, err, _, thd = cq:loop(timeout) + if not ok then + if thd then + err = debug.traceback(thd, err) + end + error(err, 2) + end + end it("invalid uris fail", function() local s = spy.new(function() end) assert(cqueues.new():wrap(function() @@ -27,4 +39,38 @@ describe("http.compat.prosody module", function() assert.same("{}", r.body) end):step()) end) + it("can perform a full request", function() + local cq = cqueues.new() + local s = server.listen { + host = "localhost"; + port = 0; + } + assert(s:listen()) + local _, host, port = s:localname() + cq:wrap(function() + s:run(function(stream) + local h = assert(stream:get_headers()) + assert.same("http", h:get ":scheme") + assert.same("GET", h:get ":method") + assert.same("/", h:get ":path") + local headers = new_headers() + headers:append(":status", "200") + headers:append("connection", "close") + assert(stream:write_headers(headers, false)) + assert(stream:write_chunk("success!", true)) + stream:shutdown() + stream.connection:shutdown() + end) + s:close() + end) + cq:wrap(function() + request(string.format("http://%s:%d", host, port), {}, function(b, c) + assert.same(200, c) + assert.same("success!", b) + s:pause() + end) + end) + assert_loop(cq, TEST_TIMEOUT) + assert.truthy(cq:empty()) + end) end) From ff046684feb47360ed28e213082210eb61515d32 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 18 Feb 2016 01:55:01 +1100 Subject: [PATCH 24/34] .luacov: Add compat source subdir --- .luacov | 1 + 1 file changed, 1 insertion(+) diff --git a/.luacov b/.luacov index 139334a8..5fd85c83 100644 --- a/.luacov +++ b/.luacov @@ -4,6 +4,7 @@ return { deletestats = true; include = { "/http/[^/]+$"; + "/http/compat/[^/]+$"; }; exclude = { }; From 9ee6ead811f8ec141a1034d5b81f5421c0e2ea0d Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 18 Feb 2016 01:59:26 +1100 Subject: [PATCH 25/34] spec/compat_prosody_spec: Add POST test --- spec/compat_prosody_spec.lua | 44 +++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/spec/compat_prosody_spec.lua b/spec/compat_prosody_spec.lua index eff11593..6b588b14 100644 --- a/spec/compat_prosody_spec.lua +++ b/spec/compat_prosody_spec.lua @@ -39,7 +39,7 @@ describe("http.compat.prosody module", function() assert.same("{}", r.body) end):step()) end) - it("can perform a full request", function() + it("can perform a GET request", function() local cq = cqueues.new() local s = server.listen { host = "localhost"; @@ -73,4 +73,46 @@ describe("http.compat.prosody module", function() assert_loop(cq, TEST_TIMEOUT) assert.truthy(cq:empty()) end) + it("can perform a POST request", function() + local cq = cqueues.new() + local s = server.listen { + host = "localhost"; + port = 0; + } + assert(s:listen()) + local _, host, port = s:localname() + cq:wrap(function() + s:run(function(stream) + local h = assert(stream:get_headers()) + assert.same("http", h:get ":scheme") + assert.same("POST", h:get ":method") + assert.same("/path", h:get ":path") + assert.same("text/plain", h:get "content-type") + local b = assert(stream:get_body_as_string()) + assert.same("this is a body", b) + local headers = new_headers() + headers:append(":status", "201") + headers:append("connection", "close") + assert(stream:write_headers(headers, false)) + assert(stream:write_chunk("success!", true)) + stream:shutdown() + stream.connection:shutdown() + end) + s:close() + end) + cq:wrap(function() + request(string.format("http://%s:%d/path", host, port), { + headers = { + ["content-type"] = "text/plain"; + }; + body = "this is a body"; + }, function(b, c) + assert.same(201, c) + assert.same("success!", b) + s:pause() + end) + end) + assert_loop(cq, TEST_TIMEOUT) + assert.truthy(cq:empty()) + end) end) From 0caa077004b2bc40c5ec5f745e6c432ed70ae3db Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 18 Feb 2016 02:04:11 +1100 Subject: [PATCH 26/34] http/compat/prosody: Remove trailing semicolons --- http/compat/prosody.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/http/compat/prosody.lua b/http/compat/prosody.lua index 6505108f..a0cf1f8c 100644 --- a/http/compat/prosody.lua +++ b/http/compat/prosody.lua @@ -19,8 +19,8 @@ local function new_prosody(url, ex, callback) local cq = assert(cqueues.running(), "must be running inside a cqueue") local ok, req = pcall(new_from_uri, url) if not ok then - callback(nil, 0, req); - return nil, "invalid-url"; + callback(nil, 0, req) + return nil, "invalid-url" end req.follow_redirects = false -- prosody doesn't follow redirects if ex then @@ -80,7 +80,7 @@ local function new_prosody(url, ex, callback) httpversion = stream.peer_version; headers = headers_as_kv; body = response_body; - }; + } callback(response_body, code, response, self) end, req) return req From 972f656b816fbaeb4a48b8274aae79e036e50c13 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Thu, 18 Feb 2016 02:11:54 +1100 Subject: [PATCH 27/34] spec/compat_prosody_spec: Add duplicate headers to POST test --- spec/compat_prosody_spec.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/compat_prosody_spec.lua b/spec/compat_prosody_spec.lua index 6b588b14..fe385159 100644 --- a/spec/compat_prosody_spec.lua +++ b/spec/compat_prosody_spec.lua @@ -93,6 +93,9 @@ describe("http.compat.prosody module", function() local headers = new_headers() headers:append(":status", "201") headers:append("connection", "close") + -- send duplicate headers + headers:append("someheader", "foo") + headers:append("someheader", "bar") assert(stream:write_headers(headers, false)) assert(stream:write_chunk("success!", true)) stream:shutdown() @@ -106,9 +109,10 @@ describe("http.compat.prosody module", function() ["content-type"] = "text/plain"; }; body = "this is a body"; - }, function(b, c) + }, function(b, c, r) assert.same(201, c) assert.same("success!", b) + assert.same("foo,bar", r.headers.someheader) s:pause() end) end) From 12c8c20ce23bd7886c66949bfac68ae6a1987804 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Sun, 28 Feb 2016 23:17:43 +1100 Subject: [PATCH 28/34] examples/h2_streaming: Commit http2 streaming example --- examples/h2_streaming.lua | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 examples/h2_streaming.lua diff --git a/examples/h2_streaming.lua b/examples/h2_streaming.lua new file mode 100644 index 00000000..9485c3c1 --- /dev/null +++ b/examples/h2_streaming.lua @@ -0,0 +1,14 @@ +--[[ +Makes a request to an HTTP2 endpoint that has an infinite length response. + +Usage: lua examples/h2_streaming.lua +]] + +local request = require "http.request" + +-- This endpoint returns a never-ending stream of chunks containing the current time +local req = request.new_from_uri("https://http2.golang.org/clockstream") +local _, stream = assert(req:go()) +for chunk in stream:each_chunk() do + io.write(chunk) +end From 78914f1456c827e318a0de9dc56e8d652dc697b3 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Tue, 1 Mar 2016 01:23:16 +1100 Subject: [PATCH 29/34] .gitignore: Actually ignore luacov files --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3e2e8052..950811f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -./luacov.report.out +/luacov.report.out +/luacov.stats.out From d60130774f87189b6e72e4fff7276c5f8c3e8cc7 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Tue, 1 Mar 2016 01:50:57 +1100 Subject: [PATCH 30/34] http/tls: Remove duplicated cipher See https://github.com/mpeterv/luacheck/issues/53#issuecomment-190222170 --- http/tls.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/http/tls.lua b/http/tls.lua index 224b3057..f15090b5 100644 --- a/http/tls.lua +++ b/http/tls.lua @@ -334,7 +334,6 @@ local spec_to_openssl = { TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 = "DHE-PSK-AES256-GCM-SHA384"; TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 = "RSA-PSK-AES128-GCM-SHA256"; TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 = "RSA-PSK-AES256-GCM-SHA384"; - TLS_PSK_WITH_AES_128_CBC_SHA256 = "PSK-AES128-CBC-SHA256"; TLS_PSK_WITH_AES_256_CBC_SHA384 = "PSK-AES256-CBC-SHA384"; TLS_PSK_WITH_NULL_SHA256 = "PSK-NULL-SHA256"; @@ -347,8 +346,6 @@ local spec_to_openssl = { TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 = "RSA-PSK-AES256-CBC-SHA384"; TLS_RSA_PSK_WITH_NULL_SHA256 = "RSA-PSK-NULL-SHA256"; TLS_RSA_PSK_WITH_NULL_SHA384 = "RSA-PSK-NULL-SHA384"; - TLS_PSK_WITH_AES_128_GCM_SHA256 = "PSK-AES128-GCM-SHA256"; - TLS_PSK_WITH_AES_256_GCM_SHA384 = "PSK-AES256-GCM-SHA384"; TLS_ECDHE_PSK_WITH_RC4_128_SHA = "ECDHE-PSK-RC4-SHA"; TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = "ECDHE-PSK-3DES-EDE-CBC-SHA"; From 96b8ae6a949c06a31268b48ec33752b679d6f6fe Mon Sep 17 00:00:00 2001 From: daurnimator Date: Tue, 1 Mar 2016 22:09:33 +1100 Subject: [PATCH 31/34] http/h1_stream: Calculate stats_sent based on uncompressed size --- http/h1_stream.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/http/h1_stream.lua b/http/h1_stream.lua index 7c796b39..78c6096a 100644 --- a/http/h1_stream.lua +++ b/http/h1_stream.lua @@ -714,6 +714,7 @@ function stream_methods:write_chunk(chunk, end_stream, timeout) else assert(self.connection.pipeline:peek() == self) end + local orig_size = #chunk if self.body_write_deflate then chunk = self.body_write_deflate(chunk, end_stream) end @@ -759,7 +760,7 @@ function stream_methods:write_chunk(chunk, end_stream, timeout) elseif self.body_write_type ~= "missing" then error("unknown body writing method") end - self.stats_sent = self.stats_sent + #chunk + self.stats_sent = self.stats_sent + orig_size if end_stream then if self.state == "half closed (remote)" then self:set_state("closed") From 985218979fc62d7e6bea7716c1c284115cb2ba77 Mon Sep 17 00:00:00 2001 From: Ryan Rion Date: Wed, 23 Mar 2016 23:10:37 -0500 Subject: [PATCH 32/34] http/cilent.lua: Fix merge with master --- http/client.lua | 32 +++++++++++++++++--------------- http/server.lua | 2 +- spec/server_spec.lua | 12 ++++-------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/http/client.lua b/http/client.lua index 94f5c610..3b6ecf71 100644 --- a/http/client.lua +++ b/http/client.lua @@ -26,21 +26,6 @@ end local function connect(options, timeout) local deadline = timeout and (monotime()+timeout) - local s do - local errno - s, errno = cs.connect { - family = options.family; - host = options.host; - port = options.port; - path = options.path; - sendname = options.sendname; - v6only = options.v6only; - nodelay = true; - } - if s == nil then - return nil, ce.strerror(errno), errno - end - end s:onerror(onerror) local tls = options.tls local version = options.version @@ -89,6 +74,23 @@ local function connect(options, timeout) end end +local function connect(options, timeout) + -- TODO: https://github.com/wahern/cqueues/issues/124 + local s, errno = cs.connect { + family = options.family; + host = options.host; + port = options.port; + path = options.path; + sendname = options.sendname; + v6only = options.v6only; + nodelay = true; + } + if s == nil then + return nil, ce.strerror(errno), errno + end + return negotiate(s, options, timeout) +end + return { connect = connect; } diff --git a/http/server.lua b/http/server.lua index bb7cc67a..02d2dd09 100644 --- a/http/server.lua +++ b/http/server.lua @@ -181,7 +181,7 @@ local function listen(tbl) local path = tbl.path assert(host or path, "need host or path") local port = tbl.port - if port == nil and tbl.host then + if port == nil and host then if tls == true then port = "443" elseif tls == false then diff --git a/spec/server_spec.lua b/spec/server_spec.lua index 5603a27f..06781d67 100644 --- a/spec/server_spec.lua +++ b/spec/server_spec.lua @@ -1,5 +1,5 @@ local TEST_TIMEOUT = 2 -describe("http.server module using hostnames", function() +describe("http.server module", function() local server = require "http.server" local client = require "http.client" local new_headers = require "http.headers".new @@ -13,12 +13,10 @@ describe("http.server module using hostnames", function() error(err, 2) end end - local socket_paths = {} local function simple_test(tls, version, path) local cq = cqueues.new() local options = {} if path then - socket_paths[#socket_paths + 1] = path options.path = path else options.host = "localhost" @@ -61,9 +59,11 @@ describe("http.server module using hostnames", function() assert(stream:write_headers(headers, true)) stream:get_headers() conn:close() - print 'closed connection // client' end) assert_loop(cq, TEST_TIMEOUT) + if path then + os.remove(path) + end assert.truthy(cq:empty()) assert.spy(on_stream).was.called() end @@ -79,7 +79,6 @@ describe("http.server module using hostnames", function() (require "http.tls".has_alpn and it or pending)("works with https 2.0 using IP", function() simple_test(true, 2.0) end) - local socket_path = os.tmpname() .. ".sock" it("works with plain http 1.1 using UNIX socket domain", function() simple_test(false, 1.1, os.tmpname() .. ".socket") end) @@ -92,7 +91,4 @@ describe("http.server module using hostnames", function() (require "http.tls".has_alpn and it or pending)("works with https 2.0 using UNIX socket domain", function() simple_test(true, 2.0, os.tmpname() .. ".socket") end) - for k, v in pairs(socket_paths) do - os.remove(v) - end end) From 85aef6fe04da198afe483c52b10ea24488dce099 Mon Sep 17 00:00:00 2001 From: Ryan Rion Date: Thu, 24 Mar 2016 00:18:54 -0500 Subject: [PATCH 33/34] spec/server_spec: Remove paths via finally() --- http/client.lua | 2 +- spec/server_spec.lua | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/http/client.lua b/http/client.lua index 3b6ecf71..8ef73b2c 100644 --- a/http/client.lua +++ b/http/client.lua @@ -24,7 +24,7 @@ local function onerror(socket, op, why, lvl) -- luacheck: ignore 212 return string.format("%s: %s", op, ce.strerror(why)), why end -local function connect(options, timeout) +local function negotiate(s, options, timeout) local deadline = timeout and (monotime()+timeout) s:onerror(onerror) local tls = options.tls diff --git a/spec/server_spec.lua b/spec/server_spec.lua index 06781d67..61a4a647 100644 --- a/spec/server_spec.lua +++ b/spec/server_spec.lua @@ -33,12 +33,10 @@ describe("http.server module", function() stream:get_headers() stream:shutdown() s:shutdown() - print 'shutting down server // spy' end) cq:wrap(function() s:run(on_stream) s:close() - print 'closed // server' end) cq:wrap(function() local options = {} @@ -61,9 +59,6 @@ describe("http.server module", function() conn:close() end) assert_loop(cq, TEST_TIMEOUT) - if path then - os.remove(path) - end assert.truthy(cq:empty()) assert.spy(on_stream).was.called() end @@ -80,15 +75,31 @@ describe("http.server module", function() simple_test(true, 2.0) end) it("works with plain http 1.1 using UNIX socket domain", function() + local path = os.tmpname() .. ".socket" simple_test(false, 1.1, os.tmpname() .. ".socket") + finally(function() + os.remove(path) + end) end) it("works with https 1.1 using UNIX socket domain", function() + local path = os.tmpname() .. ".socket" simple_test(true, 1.1, os.tmpname() .. ".socket") + finally(function() + os.remove(path) + end) end) it("works with plain http 2.0 using UNIX socket domain", function() + local path = os.tmpname() .. ".socket" simple_test(false, 2.0, os.tmpname() .. ".socket") + finally(function() + os.remove(path) + end) end); (require "http.tls".has_alpn and it or pending)("works with https 2.0 using UNIX socket domain", function() + local path = os.tmpname() .. ".socket" simple_test(true, 2.0, os.tmpname() .. ".socket") + finally(function() + os.remove(path) + end) end) end) From 80097bca2224f16f162356e65969963e6879f7dd Mon Sep 17 00:00:00 2001 From: Ryan Rion Date: Thu, 24 Mar 2016 00:33:17 -0500 Subject: [PATCH 34/34] http/server|spec/server_spec: add (pending) generation for contexts when using UNIX domain sockets --- http/server.lua | 6 +++++- spec/server_spec.lua | 8 +++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/http/server.lua b/http/server.lua index 02d2dd09..8bb4b244 100644 --- a/http/server.lua +++ b/http/server.lua @@ -192,7 +192,11 @@ local function listen(tbl) end local ctx = tbl.ctx if ctx == nil and tls ~= false then - ctx = new_ctx(host or path) + if host then + ctx = new_ctx(host) + else + error("OpenSSL ctx field expected when using UNIX domain paths") + end end local s = assert(cs.listen{ host = host; diff --git a/spec/server_spec.lua b/spec/server_spec.lua index 61a4a647..6592f3ba 100644 --- a/spec/server_spec.lua +++ b/spec/server_spec.lua @@ -22,6 +22,8 @@ describe("http.server module", function() options.host = "localhost" options.port = 0 end + options.version = version + options.tls = tls local s = server.listen(options) assert(s:listen()) local host, port @@ -81,7 +83,7 @@ describe("http.server module", function() os.remove(path) end) end) - it("works with https 1.1 using UNIX socket domain", function() + pending("works with https 1.1 using UNIX socket domain", function() local path = os.tmpname() .. ".socket" simple_test(true, 1.1, os.tmpname() .. ".socket") finally(function() @@ -94,8 +96,8 @@ describe("http.server module", function() finally(function() os.remove(path) end) - end); - (require "http.tls".has_alpn and it or pending)("works with https 2.0 using UNIX socket domain", function() + end); -- change first pending to 'it' when added ctx generation + (require "http.tls".has_alpn and pending or pending)("works with https 2.0 using UNIX socket domain", function() local path = os.tmpname() .. ".socket" simple_test(true, 2.0, os.tmpname() .. ".socket") finally(function()