diff --git a/.gitignore b/.gitignore index 3e2e8052..950811f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -./luacov.report.out +/luacov.report.out +/luacov.stats.out diff --git a/.luacov b/.luacov index 139334a8..5fd85c83 100644 --- a/.luacov +++ b/.luacov @@ -4,6 +4,7 @@ return { deletestats = true; include = { "/http/[^/]+$"; + "/http/compat/[^/]+$"; }; exclude = { }; diff --git a/.travis.yml b/.travis.yml index 1245717a..ebd1a338 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ env: - LUA="lua 5.2" ZLIB=lua-zlib - LUA="lua 5.3" - LUA="lua 5.3" ZLIB=lzlib + - LUA="lua 5.3" COMPAT53=no # lua-zlib is currently unavailable for lua5.3 https://github.com/brimworks/lua-zlib/issues/28 - LUA="luajit @" - LUA="luajit @" ZLIB=lzlib @@ -29,11 +30,12 @@ branches: before_install: - pip install hererocks - - hererocks here -r^ --$LUA # Install latest LuaRocks version + - hererocks ~/hererocks -r^ --$LUA # Install latest LuaRocks version # plus the Lua version for this build job # into 'here' subdirectory - - export PATH=$PATH:$PWD/here/bin # Add directory with all installed binaries to PATH + - export PATH=$PATH:~/hererocks/bin # Add directory with all installed binaries to PATH - eval `luarocks path --bin` + - luarocks install luacheck - luarocks install luacov-coveralls - luarocks install busted @@ -41,8 +43,10 @@ install: - luarocks install --only-deps http-scm-0.rockspec - if [ "$ZLIB" = "lzlib" ]; then luarocks install lzlib; fi - if [ "$ZLIB" = "lua-zlib" ]; then luarocks install lua-zlib; fi + - if [ "$COMPAT53" = "no" ]; then luarocks remove compat53; fi script: + - luacheck . - busted -c after_success: diff --git a/LICENSE.md b/LICENSE.md index a467327b..0adb5f6e 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 Daurnimator +Copyright (c) 2015-2016 Daurnimator Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index a8116ef1..1b6569a8 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ ## Features - Optionally asynchronous (including DNS lookups and SSL) + - Supports HTTP version 1.0, 1.1 and 2 + - Functionality for both client and server + - Websockets - Compatible with Lua 5.1, 5.2, 5.3 and [LuaJIT](http://luajit.org/) @@ -11,29 +14,22 @@ Can be found at [https://daurnimator.github.io/lua-http/](https://daurnimator.github.io/lua-http/) -# Status - -This project is a work in progress and not ready for production use. +## Status [![Build Status](https://travis-ci.org/daurnimator/lua-http.svg)](https://travis-ci.org/daurnimator/lua-http) [![Coverage Status](https://coveralls.io/repos/daurnimator/lua-http/badge.svg?branch=master&service=github)](https://coveralls.io/github/daurnimator/lua-http?branch=master) -## Todo - - - [x] HTTP 1.1 - - [x] [HTTP 2](https://http2.github.io/http2-spec/) - - [x] [HPACK](https://http2.github.io/http2-spec/compression.html) - - [ ] Connection pooling - - [ ] [`socket.http`](http://w3.impa.br/~diego/software/luasocket/http.html) compatibility layer - - [x] Prosody [`net.http`](https://prosody.im/doc/developers/net/http) compatibility layer - - [x] Handle redirects - - [ ] Be able to use an HTTP proxy - - [x] Compression (e.g. gzip) - - [ ] Websockets + - HTTP client API is reaching stability + - The HTTP server API is still changing # Installation +It's recommended to install lua-http by using [luarocks](https://luarocks.org/). +This will automatically install run-time lua dependencies for you. + + $ luarocks install --server=http://luarocks.org/dev http + ## Dependencies - [cqueues](http://25thandclement.com/~william/projects/cqueues.html) >= 20150907 @@ -42,7 +38,7 @@ This project is a work in progress and not ready for production use. - [lpeg_patterns](https://github.com/daurnimator/lpeg_patterns) >= 0.2 - [fifo](https://github.com/daurnimator/fifo.lua) -If you want to use gzip compression you will need **one** of: +To use gzip compression you need **one** of: - [lzlib](https://github.com/LuaDist/lzlib) or [lua-zlib](https://github.com/brimworks/lua-zlib) diff --git a/doc/index.md b/doc/index.md index e589ee7b..b9491d0c 100644 --- a/doc/index.md +++ b/doc/index.md @@ -664,13 +664,13 @@ Set to `math.huge` to not give up. ### `request.post301` {#http.request.post301} -Respect RFC 2616 Section 10.3.2 and **don't** convert POST requests into body-less GET requests when following a 301 redirect. The non-RFC behaviour is ubiquitous in web browsers and assumed by server. Modern HTTP endpoints send status code 308 to indicate that they don't want the method to be changed. +Respect RFC 2616 Section 10.3.2 and **don't** convert POST requests into body-less GET requests when following a 301 redirect. The non-RFC behaviour is ubiquitous in web browsers and assumed by servers. Modern HTTP endpoints send status code 308 to indicate that they don't want the method to be changed. Defaults to `false`. ### `request.post302` {#http.request.post302} -Respect RFC 2616 Section 10.3.3 and **don't** convert POST requests into body-less GET requests when following a 302 redirect. The non-RFC behaviour is ubiquitous in web browsers and assumed by server. Modern HTTP endpoints send status code 307 to indicate that they don't want the method to be changed. +Respect RFC 2616 Section 10.3.3 and **don't** convert POST requests into body-less GET requests when following a 302 redirect. The non-RFC behaviour is ubiquitous in web browsers and assumed by servers. Modern HTTP endpoints send status code 307 to indicate that they don't want the method to be changed. Defaults to `false`. @@ -683,6 +683,15 @@ Allows setting a request body. `body` may be a string, function or lua file obje - If `body` is a lua file object, it will be [`:seek`'d](http://www.lua.org/manual/5.3/manual.html#pdf-file:seek) to the start, then sent as a body. Any errors encountered during file operations **will be thrown**. +### `request:clone()` {#http.request:clone} + +Creates and returns a clone of the request. + +The clone has its own deep copy of the [`.headers`](#http.request.headers) field. + +The [`.tls`](#http.request.tls) and body fields are shallow copied from the original request. + + ### `request:go(timeout)` {#http.request:timeout} Performs the request. @@ -693,7 +702,26 @@ On success, returns the response [*headers*](#http.headers) and a [*stream*](#st ## http.server -### `listen(options)` {#http.client.connect} +This interface is **unstable**. + +### `listen(options)` {#http.server.connect} + + +### `server:listen(timeout)` {#http.server:listen} + + +### `server:localname()` {#http.server:localname} + + +### `server:pause()` {#http.server:pause} + +Cause [`server:run`](#http.server:run) to stop processing new clients and return. + + +### `server:close()` {#http.server:close} + + +### `server:run(on_stream, cq)` {#http.server:run} ## http.stream_common diff --git a/examples/h2_streaming.lua b/examples/h2_streaming.lua new file mode 100644 index 00000000..9485c3c1 --- /dev/null +++ b/examples/h2_streaming.lua @@ -0,0 +1,14 @@ +--[[ +Makes a request to an HTTP2 endpoint that has an infinite length response. + +Usage: lua examples/h2_streaming.lua +]] + +local request = require "http.request" + +-- This endpoint returns a never-ending stream of chunks containing the current time +local req = request.new_from_uri("https://http2.golang.org/clockstream") +local _, stream = assert(req:go()) +for chunk in stream:each_chunk() do + io.write(chunk) +end diff --git a/http/client.lua b/http/client.lua index 7cda2582..8ef73b2c 100644 --- a/http/client.lua +++ b/http/client.lua @@ -24,22 +24,8 @@ local function onerror(socket, op, why, lvl) -- luacheck: ignore 212 return string.format("%s: %s", op, ce.strerror(why)), why end -local function connect(options, timeout) +local function negotiate(s, options, timeout) local deadline = timeout and (monotime()+timeout) - local s do - local errno - s, errno = cs.connect { - family = options.family; - host = options.host; - port = options.port; - sendname = options.sendname; - v6only = options.v6only; - nodelay = true; - } - if s == nil then - return nil, ce.strerror(errno), errno - end - end s:onerror(onerror) local tls = options.tls local version = options.version @@ -88,6 +74,23 @@ local function connect(options, timeout) end end +local function connect(options, timeout) + -- TODO: https://github.com/wahern/cqueues/issues/124 + local s, errno = cs.connect { + family = options.family; + host = options.host; + port = options.port; + path = options.path; + sendname = options.sendname; + v6only = options.v6only; + nodelay = true; + } + if s == nil then + return nil, ce.strerror(errno), errno + end + return negotiate(s, options, timeout) +end + return { connect = connect; } diff --git a/http/compat/prosody.lua b/http/compat/prosody.lua index 6505108f..a0cf1f8c 100644 --- a/http/compat/prosody.lua +++ b/http/compat/prosody.lua @@ -19,8 +19,8 @@ local function new_prosody(url, ex, callback) local cq = assert(cqueues.running(), "must be running inside a cqueue") local ok, req = pcall(new_from_uri, url) if not ok then - callback(nil, 0, req); - return nil, "invalid-url"; + callback(nil, 0, req) + return nil, "invalid-url" end req.follow_redirects = false -- prosody doesn't follow redirects if ex then @@ -80,7 +80,7 @@ local function new_prosody(url, ex, callback) httpversion = stream.peer_version; headers = headers_as_kv; body = response_body; - }; + } callback(response_body, code, response, self) end, req) return req diff --git a/http/h1_connection.lua b/http/h1_connection.lua index 2fcf7ce7..0f60ea29 100644 --- a/http/h1_connection.lua +++ b/http/h1_connection.lua @@ -120,22 +120,25 @@ end -- this function *should never throw* function connection_methods:get_next_incoming_stream(timeout) assert(self.type == "server") + local deadline = timeout and (monotime()+timeout) -- Make sure we don't try and read before the previous request has been fully read - if self.req_locked then + repeat -- Wait until previous requests have been fully read - if not self.req_cond:wait(timeout) then - return nil, ce.ETIMEDOUT + if self.req_locked then + if not self.req_cond:wait(deadline and deadline - monotime()) then + return nil, ce.ETIMEDOUT + end + assert(self.req_locked == nil) end - assert(self.req_locked == nil) - end - if self.socket == nil or self.socket:eof("r") then - return nil, ce.EPIPE - end - -- check if socket has already got an error set - local errno = self.socket:error("r") - if errno then - return nil, onerror(self.socket, "read", errno, 3) - end + if self.socket == nil then + return nil, ce.EPIPE + end + -- Wait for at least one byte + local ok, err, errno = self.socket:fill(1, deadline and deadline-monotime()) + if not ok then + return nil, err or ce.EPIPE, errno + end + until not self.req_locked local stream = h1_stream.new(self) self.pipeline:push(stream) self.req_locked = stream diff --git a/http/h1_stream.lua b/http/h1_stream.lua index 8d648a82..78c6096a 100644 --- a/http/h1_stream.lua +++ b/http/h1_stream.lua @@ -99,11 +99,11 @@ function stream_methods:set_state(new) -- If we have just finished writing the response if (old == "idle" or old == "open" or old == "half closed (remote)") and (new == "half closed (local)" or new == "closed") then + -- remove ourselves from the write pipeline + assert(self.connection.pipeline:pop() == self) if self.close_when_done then self.connection:shutdown() end - -- remove ourselves from the write pipeline - assert(self.connection.pipeline:pop() == self) local next_stream = self.connection.pipeline:peek() if next_stream then next_stream.pipeline_cond:signal() @@ -124,11 +124,11 @@ function stream_methods:set_state(new) -- If we have just finished reading the response; if (old == "idle" or old == "open" or old == "half closed (local)") and (new == "half closed (remote)" or new == "closed") then + -- remove ourselves from the read pipeline + assert(self.connection.pipeline:pop() == self) if self.close_when_done then self.connection:shutdown() end - -- remove ourselves from the read pipeline - assert(self.connection.pipeline:pop() == self) local next_stream = self.connection.pipeline:peek() if next_stream then next_stream.pipeline_cond:signal() @@ -370,6 +370,9 @@ function stream_methods:write_headers(headers, end_stream, timeout) end local status_code, method if self.type == "server" then + if self.state == "idle" then + error("cannot write headers when stream is idle") + end -- Make sure we're at the front of the pipeline if self.connection.pipeline:peek() ~= self then if not self.pipeline_cond:wait(deadline and (deadline-monotime)) then @@ -711,6 +714,7 @@ function stream_methods:write_chunk(chunk, end_stream, timeout) else assert(self.connection.pipeline:peek() == self) end + local orig_size = #chunk if self.body_write_deflate then chunk = self.body_write_deflate(chunk, end_stream) end @@ -756,7 +760,7 @@ function stream_methods:write_chunk(chunk, end_stream, timeout) elseif self.body_write_type ~= "missing" then error("unknown body writing method") end - self.stats_sent = self.stats_sent + #chunk + self.stats_sent = self.stats_sent + orig_size if end_stream then if self.state == "half closed (remote)" then self:set_state("closed") diff --git a/http/h2_connection.lua b/http/h2_connection.lua index d20d9a57..4a0e8a21 100644 --- a/http/h2_connection.lua +++ b/http/h2_connection.lua @@ -9,10 +9,14 @@ local h2_error = require "http.h2_error" local h2_stream = require "http.h2_stream" local hpack = require "http.hpack" local h2_banned_ciphers = require "http.tls".banned_ciphers -local assert = require "compat53.module".assert local spack = string.pack or require "compat53.string".pack local sunpack = string.unpack or require "compat53.string".unpack +local assert = assert +if _VERSION:match("%d+%.?%d*") < "5.3" then + assert = require "compat53.module".assert +end + local function xor(a, b) return (a and b) or not (a or b) end diff --git a/http/h2_stream.lua b/http/h2_stream.lua index 66ffdcda..d74911ff 100644 --- a/http/h2_stream.lua +++ b/http/h2_stream.lua @@ -7,11 +7,15 @@ local band = require "http.bit".band local bor = require "http.bit".bor local h2_errors = require "http.h2_error".errors local stream_common = require "http.stream_common" -local assert = require "compat53.module".assert local spack = string.pack or require "compat53.string".pack local sunpack = string.unpack or require "compat53.string".unpack local unpack = table.unpack or unpack -- luacheck: ignore 113 +local assert = assert +if _VERSION:match("%d+%.?%d*") < "5.3" then + assert = require "compat53.module".assert +end + local function xor(a, b) return (a and b) or not (a or b) end diff --git a/http/request.lua b/http/request.lua index 0a8d013a..e200a38b 100644 --- a/http/request.lua +++ b/http/request.lua @@ -117,6 +117,25 @@ local function new_from_stream(stream) return self end +function request_methods:clone() + return setmetatable({ + host = self.host; + port = self.port; + tls = self.tls; + sendname = self.sendname; + version = self.version; + + headers = self.headers:clone(); + body = self.body; + + expect_100_timeout = rawget(self, "expect_100_timeout"); + follow_redirects = rawget(self, "follow_redirects"); + max_redirects = rawget(self, "max_redirects"); + post301 = rawget(self, "post301"); + post302 = rawget(self, "post302"); + }, request_mt) +end + function request_methods:to_url() -- TODO: userinfo section (username/password) local method = self.headers:get(":method") @@ -160,37 +179,65 @@ function request_methods:handle_redirect(orig_headers) return nil, "missing location header for redirect", ce.EINVAL end local uri_t = assert(uri_ref:match(location), "invalid URI") - local orig_scheme = self.headers:get(":scheme") - if uri_t.scheme == nil then - uri_t.scheme = orig_scheme + local new_req = self:clone() + new_req.max_redirects = max_redirects - 1 + local is_connect = new_req.headers:get(":method") == "CONNECT" + if uri_t.scheme ~= nil then + if not is_connect then + new_req.headers:upsert(":scheme", uri_t.scheme) + end + if uri_t.scheme == "https" or uri_t.scheme == "wss" then + new_req.tls = self.tls or true + else + new_req.tls = false + end + end + if uri_t.host ~= nil then + local new_scheme = new_req.headers:get(":scheme") + new_req.host = uri_t.host + new_req.port = uri_t.port or http_util.scheme_to_port[new_scheme] + if not is_connect then + new_req.headers:upsert(":authority", http_util.to_authority(uri_t.host, uri_t.port, new_scheme)) + end + new_req.sendname = nil end - if uri_t.host == nil then - uri_t.host, uri_t.port = http_util.split_authority(self.headers:get(":authority"), orig_scheme) + if is_connect then + assert(uri_t.path == "", "CONNECT requests cannot have a path") + assert(uri_t.query == nil, "CONNECT requests cannot have a query") + else + local new_path + if uri_t.path == "" then + new_path = "/" + else + new_path = http_util.encodeURI(uri_t.path) + if new_path:sub(1, 1) ~= "/" then -- relative path + local orig_target = self.headers:get(":path") + local orig_path = assert(uri_ref:match(orig_target)).path + orig_path = http_util.encodeURI(orig_path) + new_path = http_util.resolve_relative_path(orig_path, new_path) + end + end + if uri_t.query then + new_path = new_path .. "?" .. http_util.encodeURI(uri_t.query) + end + new_req.headers:upsert(":path", new_path) end - if uri_t.path ~= nil then - uri_t.path = http_util.encodeURI(uri_t.path) - if uri_t.path:sub(1, 1) ~= "/" then -- relative path - local orig_target = self.headers:get(":path") - local orig_path = assert(uri_ref:match(orig_target)).path - orig_path = http_util.encodeURI(orig_path) - uri_t.path = http_util.resolve_relative_path(orig_path, uri_t.path) + if uri_t.userinfo then + local field + if is_connect then + field = "proxy-authorization" + else + field = "authorization" end + new_req.headers:upsert(field, "basic " .. basexx.to_base64(uri_t.userinfo), true) end - local headers = self.headers:clone() - local new_req = new_from_uri_t(uri_t, headers) - new_req.expect_100_timeout = rawget(self, "expect_100_timeout") - new_req.follow_redirects = rawget(self, "follow_redirects") - new_req.max_redirects = max_redirects - 1 - new_req.post301 = rawget(self, "post301") - new_req.post302 = rawget(self, "post302") if not new_req.tls and self.tls then --[[ RFC 7231 5.5.2: A user agent MUST NOT send a Referer header field in an unsecured HTTP request if the referring page was received with a secure protocol.]] - headers:delete("referer") + new_req.headers:delete("referer") else - headers:upsert("referer", self:to_url()) + new_req.headers:upsert("referer", self:to_url()) end - new_req.body = self.body -- Change POST requests to a body-less GET on redirect? local orig_status = orig_headers:get(":status") if (orig_status == "303" @@ -198,19 +245,19 @@ function request_methods:handle_redirect(orig_headers) or (orig_status == "302" and not self.post302) ) and self.headers:get(":method") == "POST" then - headers:upsert(":method", "GET") + new_req.headers:upsert(":method", "GET") -- Remove headers that don't make sense without a body -- Headers that require a body - headers:delete("transfer-encoding") - headers:delete("content-length") + new_req.headers:delete("transfer-encoding") + new_req.headers:delete("content-length") -- Representation Metadata from RFC 7231 Section 3.1 - headers:delete("content-encoding") - headers:delete("content-language") - headers:delete("content-location") - headers:delete("content-type") + new_req.headers:delete("content-encoding") + new_req.headers:delete("content-language") + new_req.headers:delete("content-location") + new_req.headers:delete("content-type") -- Other... - if headers:get("expect") == "100-continue" then - headers:delete("expect") + if new_req.headers:get("expect") == "100-continue" then + new_req.headers:delete("expect") end new_req.body = nil end diff --git a/http/server.lua b/http/server.lua index abaa9f7f..8bb4b244 100644 --- a/http/server.lua +++ b/http/server.lua @@ -161,8 +161,9 @@ local server_mt = { --[[ Starts listening on the given socket Takes a table of options: - - `.host`: address to bind to (required) + - `.host`: address to bind to (required if not `.path`) - `.port`: port to bind to (optional if tls isn't `nil`, in which case defaults to 80 for `.tls == false` or 443 if `.tls == true`) + - `.path`: path to a UNIX socket to bind to (required if not `.host`) - `.v6only`: allow ipv6 only (no ipv4-mapped-ipv6) - `.reuseaddr`: turn on SO_REUSEADDR flag? - `.reuseport`: turn on SO_REUSEPORT flag? @@ -176,9 +177,11 @@ Takes a table of options: ]] local function listen(tbl) local tls = tbl.tls - local host = assert(tbl.host, "need host") + local host = tbl.host + local path = tbl.path + assert(host or path, "need host or path") local port = tbl.port - if port == nil then + if port == nil and host then if tls == true then port = "443" elseif tls == false then @@ -189,11 +192,16 @@ local function listen(tbl) end local ctx = tbl.ctx if ctx == nil and tls ~= false then - ctx = new_ctx(host) + if host then + ctx = new_ctx(host) + else + error("OpenSSL ctx field expected when using UNIX domain paths") + end end local s = assert(cs.listen{ host = host; port = port; + path = path; v6only = tbl.v6only; reuseaddr = tbl.reuseaddr; reuseport = tbl.reuseport; @@ -209,6 +217,7 @@ local function listen(tbl) ctx = ctx; max_concurrent = tbl.max_concurrent; n_connections = 0; + shutdown_cond = cc.new(); connection_done = cc.new(); -- signalled when connection has been closed client_timeout = tbl.client_timeout; }, server_mt) @@ -224,8 +233,13 @@ function server_methods:localname() return self.socket:localname() end +-- Shutdown the server socket +-- does *not* shutdown connections that originated on this socket function server_methods:shutdown() - self.socket:shutdown() + if self.shutdown_cond then + self.shutdown_cond:signal() + end + self.shutdown_cond = nil end function server_methods:close() @@ -235,19 +249,20 @@ function server_methods:close() self.socket:close() end --- accepts a new client and returns it as an http connection object function server_methods:run(on_stream, cq) cq = cq or cqueues.running() - while true do + while self.shutdown_cond do if self.n_connections >= self.max_concurrent then self.connection_done:wait() end -- Yield this thread until a client arrives - local socket, accept_err = self.socket:accept{nodelay = true;} + local socket, accept_err = self.socket:accept({nodelay = true;}, 0) if socket == nil then - if accept_err == ce.EINVAL then - -- has been shutdown - break + if accept_err == ce.ETIMEDOUT then + -- Wait for a client + if cqueues.poll(self.socket, self.shutdown_cond) == self.shutdown_cond then + break + end elseif accept_err == ce.EMFILE then -- Wait for another request to finish if not self.connection_done:wait(0.1) then diff --git a/http/tls.lua b/http/tls.lua index 224b3057..f15090b5 100644 --- a/http/tls.lua +++ b/http/tls.lua @@ -334,7 +334,6 @@ local spec_to_openssl = { TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 = "DHE-PSK-AES256-GCM-SHA384"; TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 = "RSA-PSK-AES128-GCM-SHA256"; TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 = "RSA-PSK-AES256-GCM-SHA384"; - TLS_PSK_WITH_AES_128_CBC_SHA256 = "PSK-AES128-CBC-SHA256"; TLS_PSK_WITH_AES_256_CBC_SHA384 = "PSK-AES256-CBC-SHA384"; TLS_PSK_WITH_NULL_SHA256 = "PSK-NULL-SHA256"; @@ -347,8 +346,6 @@ local spec_to_openssl = { TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 = "RSA-PSK-AES256-CBC-SHA384"; TLS_RSA_PSK_WITH_NULL_SHA256 = "RSA-PSK-NULL-SHA256"; TLS_RSA_PSK_WITH_NULL_SHA384 = "RSA-PSK-NULL-SHA384"; - TLS_PSK_WITH_AES_128_GCM_SHA256 = "PSK-AES128-GCM-SHA256"; - TLS_PSK_WITH_AES_256_GCM_SHA384 = "PSK-AES256-GCM-SHA384"; TLS_ECDHE_PSK_WITH_RC4_128_SHA = "ECDHE-PSK-RC4-SHA"; TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = "ECDHE-PSK-3DES-EDE-CBC-SHA"; diff --git a/spec/compat_prosody_spec.lua b/spec/compat_prosody_spec.lua index e3bcdefd..fe385159 100644 --- a/spec/compat_prosody_spec.lua +++ b/spec/compat_prosody_spec.lua @@ -1,6 +1,18 @@ +local TEST_TIMEOUT = 2 describe("http.compat.prosody module", function() local cqueues = require "cqueues" local request = require "http.compat.prosody".request + local new_headers = require "http.headers".new + local server = require "http.server" + local function assert_loop(cq, timeout) + local ok, err, _, thd = cq:loop(timeout) + if not ok then + if thd then + err = debug.traceback(thd, err) + end + error(err, 2) + end + end it("invalid uris fail", function() local s = spy.new(function() end) assert(cqueues.new():wrap(function() @@ -27,4 +39,84 @@ describe("http.compat.prosody module", function() assert.same("{}", r.body) end):step()) end) + it("can perform a GET request", function() + local cq = cqueues.new() + local s = server.listen { + host = "localhost"; + port = 0; + } + assert(s:listen()) + local _, host, port = s:localname() + cq:wrap(function() + s:run(function(stream) + local h = assert(stream:get_headers()) + assert.same("http", h:get ":scheme") + assert.same("GET", h:get ":method") + assert.same("/", h:get ":path") + local headers = new_headers() + headers:append(":status", "200") + headers:append("connection", "close") + assert(stream:write_headers(headers, false)) + assert(stream:write_chunk("success!", true)) + stream:shutdown() + stream.connection:shutdown() + end) + s:close() + end) + cq:wrap(function() + request(string.format("http://%s:%d", host, port), {}, function(b, c) + assert.same(200, c) + assert.same("success!", b) + s:pause() + end) + end) + assert_loop(cq, TEST_TIMEOUT) + assert.truthy(cq:empty()) + end) + it("can perform a POST request", function() + local cq = cqueues.new() + local s = server.listen { + host = "localhost"; + port = 0; + } + assert(s:listen()) + local _, host, port = s:localname() + cq:wrap(function() + s:run(function(stream) + local h = assert(stream:get_headers()) + assert.same("http", h:get ":scheme") + assert.same("POST", h:get ":method") + assert.same("/path", h:get ":path") + assert.same("text/plain", h:get "content-type") + local b = assert(stream:get_body_as_string()) + assert.same("this is a body", b) + local headers = new_headers() + headers:append(":status", "201") + headers:append("connection", "close") + -- send duplicate headers + headers:append("someheader", "foo") + headers:append("someheader", "bar") + assert(stream:write_headers(headers, false)) + assert(stream:write_chunk("success!", true)) + stream:shutdown() + stream.connection:shutdown() + end) + s:close() + end) + cq:wrap(function() + request(string.format("http://%s:%d/path", host, port), { + headers = { + ["content-type"] = "text/plain"; + }; + body = "this is a body"; + }, function(b, c, r) + assert.same(201, c) + assert.same("success!", b) + assert.same("foo,bar", r.headers.someheader) + s:pause() + end) + end) + assert_loop(cq, TEST_TIMEOUT) + assert.truthy(cq:empty()) + end) end) diff --git a/spec/h1_connection_spec.lua b/spec/h1_connection_spec.lua index 47b22a0c..95c3ddad 100644 --- a/spec/h1_connection_spec.lua +++ b/spec/h1_connection_spec.lua @@ -402,7 +402,6 @@ describe("low level http 1 connection operations", function() end) describe("high level http1 connection operations", function() local h1_connection = require "http.h1_connection" - local cqueues = require "cqueues" local cs = require "cqueues.socket" local ce = require "cqueues.errno" @@ -415,16 +414,7 @@ describe("high level http1 connection operations", function() it(":get_next_incoming_stream times out", function() local s, c = new_pair(1.1) -- luacheck: ignore 211 - local cq = cqueues.new() - cq:wrap(function() - local stream = s:get_next_incoming_stream() - cq:wrap(function() - assert.same({nil, ce.ETIMEDOUT}, {s:get_next_incoming_stream(0.05)}) - end) - cqueues.sleep(0.1) - stream:shutdown() - end) - assert(cq:loop()) + assert.same({nil, ce.ETIMEDOUT}, {s:get_next_incoming_stream(0.05)}) end) it(":get_next_incoming_stream returns nil, EPIPE when no data", function() local s, c = new_pair(1.1) diff --git a/spec/request_spec.lua b/spec/request_spec.lua index 657fb0b3..6b04ed90 100644 --- a/spec/request_spec.lua +++ b/spec/request_spec.lua @@ -119,81 +119,120 @@ describe("http.request module", function() req:set_body(io.tmpfile()) assert.same("100-continue", req.headers:get("expect")) end) + local headers = require "http.headers" it(":handle_redirect works", function() - local headers = require "http.headers" - do - local orig_req = request.new_from_uri("http://example.com") - local orig_headers = headers.new() - orig_headers:append(":status", "301") - orig_headers:append("location", "/foo") - local new_req = orig_req:handle_redirect(orig_headers) - -- same - assert.same(orig_req.host, new_req.host) - assert.same(orig_req.port, new_req.port) - assert.same(orig_req.tls, new_req.tls) - assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") - assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") - assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") - assert.same(orig_req.body, new_req.body) - -- different - assert.same("/foo", new_req.headers:get ":path") - assert.same(orig_req.max_redirects-1, new_req.max_redirects) - end - do - local orig_req = request.new_from_uri("http://example.com") - local orig_headers = headers.new() - orig_headers:append(":status", "302") - orig_headers:append("location", "//blah.com:1234/example") - local new_req = orig_req:handle_redirect(orig_headers) - -- same - assert.same(orig_req.tls, new_req.tls) - assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") - assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") - assert.same(orig_req.body, new_req.body) - -- different - assert.same("blah.com", new_req.host) - assert.same(1234, new_req.port) - assert.same("blah.com:1234", new_req.headers:get ":authority") - assert.same("/example", new_req.headers:get ":path") - assert.same(orig_req.max_redirects-1, new_req.max_redirects) - end - do -- maximum redirects exceeded - local ce = require "cqueues.errno" - local orig_req = request.new_from_uri("http://example.com") - orig_req.max_redirects = 0 - local orig_headers = headers.new() - orig_headers:append(":status", "302") - orig_headers:append("location", "/") - assert.same({nil, "maximum redirects exceeded", ce.ELOOP}, {orig_req:handle_redirect(orig_headers)}) - end - do -- missing location header - local ce = require "cqueues.errno" - local orig_req = request.new_from_uri("http://example.com") - local orig_headers = headers.new() - orig_headers:append(":status", "302") - assert.same({nil, "missing location header for redirect", ce.EINVAL}, {orig_req:handle_redirect(orig_headers)}) - end - do -- POST => GET transformation - local orig_req = request.new_from_uri("http://example.com") - orig_req.headers:upsert(":method", "POST") - orig_req.headers:upsert("content-type", "text/plain") - orig_req:set_body("foo") - local orig_headers = headers.new() - orig_headers:append(":status", "303") - orig_headers:append("location", "/foo") - local new_req = orig_req:handle_redirect(orig_headers) - -- same - assert.same(orig_req.host, new_req.host) - assert.same(orig_req.port, new_req.port) - assert.same(orig_req.tls, new_req.tls) - assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") - assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") - -- different - assert.same("GET", new_req.headers:get ":method") - assert.same("/foo", new_req.headers:get ":path") - assert.falsy(new_req.headers:has "content-type") - assert.same(nil, new_req.body) - assert.same(orig_req.max_redirects-1, new_req.max_redirects) - end + local orig_req = request.new_from_uri("http://example.com") + local orig_headers = headers.new() + orig_headers:append(":status", "301") + orig_headers:append("location", "/foo") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.host, new_req.host) + assert.same(orig_req.port, new_req.port) + assert.same(orig_req.tls, new_req.tls) + assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") + assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") + assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") + assert.same(orig_req.body, new_req.body) + -- different + assert.same("/foo", new_req.headers:get ":path") + assert.same(orig_req.max_redirects-1, new_req.max_redirects) + end) + it(":handle_redirect works with cross-scheme port-less uri", function() + local orig_req = request.new_from_uri("http://example.com") + local orig_headers = headers.new() + orig_headers:append(":status", "302") + orig_headers:append("location", "https://blah.com/example") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") + assert.same(orig_req.body, new_req.body) + -- different + assert.same(false, orig_req.tls) + assert.same(true, new_req.tls) + assert.same("https", new_req.headers:get ":scheme") + assert.same("blah.com", new_req.host) + assert.same(80, orig_req.port) + assert.same(443, new_req.port) + assert.same("blah.com", new_req.headers:get ":authority") + assert.same("/example", new_req.headers:get ":path") + assert.same(orig_req.max_redirects-1, new_req.max_redirects) + end) + it(":handle_redirect works with scheme relative uri", function() + local orig_req = request.new_from_uri("http://example.com") + local orig_headers = headers.new() + orig_headers:append(":status", "302") + orig_headers:append("location", "//blah.com:1234/example") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.tls, new_req.tls) + assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") + assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") + assert.same(orig_req.body, new_req.body) + -- different + assert.same("blah.com", new_req.host) + assert.same(1234, new_req.port) + assert.same("blah.com:1234", new_req.headers:get ":authority") + assert.same("/example", new_req.headers:get ":path") + assert.same(orig_req.max_redirects-1, new_req.max_redirects) + end) + it(":handle_redirect detects maximum redirects exceeded", function() + local ce = require "cqueues.errno" + local orig_req = request.new_from_uri("http://example.com") + orig_req.max_redirects = 0 + local orig_headers = headers.new() + orig_headers:append(":status", "302") + orig_headers:append("location", "/") + assert.same({nil, "maximum redirects exceeded", ce.ELOOP}, {orig_req:handle_redirect(orig_headers)}) + end) + it(":handle_redirect detects missing location header", function() + local ce = require "cqueues.errno" + local orig_req = request.new_from_uri("http://example.com") + local orig_headers = headers.new() + orig_headers:append(":status", "302") + assert.same({nil, "missing location header for redirect", ce.EINVAL}, {orig_req:handle_redirect(orig_headers)}) + end) + it(":handle_redirect detects POST => GET transformation", function() + local orig_req = request.new_from_uri("http://example.com") + orig_req.headers:upsert(":method", "POST") + orig_req.headers:upsert("content-type", "text/plain") + orig_req:set_body("foo") + local orig_headers = headers.new() + orig_headers:append(":status", "303") + orig_headers:append("location", "/foo") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.host, new_req.host) + assert.same(orig_req.port, new_req.port) + assert.same(orig_req.tls, new_req.tls) + assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") + assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") + -- different + assert.same("GET", new_req.headers:get ":method") + assert.same("/foo", new_req.headers:get ":path") + assert.falsy(new_req.headers:has "content-type") + assert.same(nil, new_req.body) + assert.same(orig_req.max_redirects-1, new_req.max_redirects) + end) + it(":handle_redirect deletes keeps original custom host, port and sendname if relative", function() + local orig_req = request.new_from_uri("http://example.com") + orig_req.host = "other.com" + orig_req.sendname = "something.else" + local orig_headers = headers.new() + orig_headers:append(":status", "301") + orig_headers:append("location", "/foo") + local new_req = orig_req:handle_redirect(orig_headers) + -- same + assert.same(orig_req.host, new_req.host) + assert.same(orig_req.port, new_req.port) + assert.same(orig_req.tls, new_req.tls) + assert.same(orig_req.sendname, new_req.sendname) + assert.same(orig_req.headers:get ":authority", new_req.headers:get ":authority") + assert.same(orig_req.headers:get ":method", new_req.headers:get ":method") + assert.same(orig_req.headers:get ":scheme", new_req.headers:get ":scheme") + assert.same(orig_req.body, new_req.body) + -- different + assert.same("/foo", new_req.headers:get ":path") + assert.same(orig_req.max_redirects-1, new_req.max_redirects) end) end) diff --git a/spec/server_spec.lua b/spec/server_spec.lua index ae50ad19..6592f3ba 100644 --- a/spec/server_spec.lua +++ b/spec/server_spec.lua @@ -13,14 +13,24 @@ describe("http.server module", function() error(err, 2) end end - local function simple_test(tls, version) + local function simple_test(tls, version, path) local cq = cqueues.new() - local s = server.listen { - host = "localhost"; - port = 0; - } + local options = {} + if path then + options.path = path + else + options.host = "localhost" + options.port = 0 + end + options.version = version + options.tls = tls + local s = server.listen(options) assert(s:listen()) - local _, host, port = s:localname() + local host, port + if not path then + local _ + _, host, port = s:localname() + end local on_stream = spy.new(function(stream) stream:get_headers() stream:shutdown() @@ -31,12 +41,16 @@ describe("http.server module", function() s:close() end) cq:wrap(function() - local conn = client.connect { - host = host; - port = port; - tls = tls; - version = version; - } + local options = {} + if path then + options.path = path + else + options.host = host + options.port = port + end + options.tls = tls + options.version = version + local conn = client.connect(options) local stream = conn:new_stream() local headers = new_headers() headers:append(":method", "GET") @@ -50,16 +64,44 @@ describe("http.server module", function() assert.truthy(cq:empty()) assert.spy(on_stream).was.called() end - it("works with plain http 1.1", function() + it("works with plain http 1.1 using IP", function() simple_test(false, 1.1) end) - it("works with https 1.1", function() + it("works with https 1.1 using IP", function() simple_test(true, 1.1) end) - it("works with plain http 2.0", function() + it("works with plain http 2.0 using IP", function() simple_test(false, 2.0) end); - (require "http.tls".has_alpn and it or pending)("works with https 2.0", function() + (require "http.tls".has_alpn and it or pending)("works with https 2.0 using IP", function() simple_test(true, 2.0) end) + it("works with plain http 1.1 using UNIX socket domain", function() + local path = os.tmpname() .. ".socket" + simple_test(false, 1.1, os.tmpname() .. ".socket") + finally(function() + os.remove(path) + end) + end) + pending("works with https 1.1 using UNIX socket domain", function() + local path = os.tmpname() .. ".socket" + simple_test(true, 1.1, os.tmpname() .. ".socket") + finally(function() + os.remove(path) + end) + end) + it("works with plain http 2.0 using UNIX socket domain", function() + local path = os.tmpname() .. ".socket" + simple_test(false, 2.0, os.tmpname() .. ".socket") + finally(function() + os.remove(path) + end) + end); -- change first pending to 'it' when added ctx generation + (require "http.tls".has_alpn and pending or pending)("works with https 2.0 using UNIX socket domain", function() + local path = os.tmpname() .. ".socket" + simple_test(true, 2.0, os.tmpname() .. ".socket") + finally(function() + os.remove(path) + end) + end) end) diff --git a/spec/util_spec.lua b/spec/util_spec.lua index 74deb6ef..457965f6 100644 --- a/spec/util_spec.lua +++ b/spec/util_spec.lua @@ -1,6 +1,9 @@ describe("http.util module", function() local unpack = table.unpack or unpack -- luacheck: ignore 113 local util = require "http.util" + it("decodeURI works", function() + assert.same("Encoded string", util.decodeURI("Encoded%20string")) + end) it("decodeURI doesn't decode blacklisted characters", function() assert.same("%24", util.decodeURI("%24")) local s = util.encodeURIComponent("#$&+,/:;=?@") @@ -53,6 +56,23 @@ describe("http.util module", function() assert.same(t, r) end end) + it("split_authority works", function() + assert.same({"example.com", 80}, {util.split_authority("example.com", "http")}) + assert.same({"example.com", 8000}, {util.split_authority("example.com:8000", "http")}) + assert.has.errors(function() + util.split_authority("example.com", "madeupscheme") + end) + -- IPv6 + assert.same({"::1", 443}, {util.split_authority("[::1]", "https")}) + assert.same({"::1", 8000}, {util.split_authority("[::1]:8000", "https")}) + end) + it("to_authority works", function() + assert.same("example.com", util.to_authority("example.com", 80, "http")) + assert.same("example.com:8000", util.to_authority("example.com", 8000, "http")) + -- IPv6 + assert.same("[::1]", util.to_authority("::1", 443, "https")) + assert.same("[::1]:8000", util.to_authority("::1", 8000, "https")) + end) it("split_header works correctly", function() -- nil assert.same({n=0}, util.split_header(nil))