diff --git a/.travis.yml b/.travis.yml index ff013102d..eb59f5b7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ addons: - doxygen - mkdocs - graphviz + - zlib1g-dev before_install: - if [ "$TRAVIS_COMPILER" == "gcc" -a "$TRAVIS_CPU_ARCH" == "amd64" ]; then export PUSH_COVERAGE=ON; fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 897253d87..7cc1c2a7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,11 +6,14 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") find_package(Tcmalloc) find_package(Threads) find_package(OpenSSL) +find_package(ZLIB REQUIRED) if(OPENSSL_FOUND) include_directories(${OPENSSL_INCLUDE_DIR}) endif() +include_directories(${ZLIB_INCLUDE_DIR}) + find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) message("Found ccache ${CCACHE_FOUND}") diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 505a8a691..fe60eef20 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -5,20 +5,34 @@ if (MSVC) add_executable(example_vs example_vs.cpp) target_link_libraries(example_vs ${Boost_LIBRARIES}) target_link_libraries(example_vs ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(example_vs ${ZLIB_LIBRARIES}) else () add_executable(helloworld helloworld.cpp) target_link_libraries(helloworld ${Boost_LIBRARIES}) target_link_libraries(helloworld ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(helloworld ${ZLIB_LIBRARIES}) + +add_executable(example_static_file example_static_file.cpp) +target_link_libraries(example_static_file ${Boost_LIBRARIES}) +target_link_libraries(example_static_file ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(example_static_file ${ZLIB_LIBRARIES}) + +add_executable(example_compression example_compression.cpp) +target_link_libraries(example_compression ${Boost_LIBRARIES}) +target_link_libraries(example_compression ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(example_compression ${ZLIB_LIBRARIES}) if (OPENSSL_FOUND) add_executable(example_ssl ssl/example_ssl.cpp) target_link_libraries(example_ssl ${Boost_LIBRARIES}) target_link_libraries(example_ssl ${CMAKE_THREAD_LIBS_INIT} ${OPENSSL_LIBRARIES}) + target_link_libraries(example_ssl ${ZLIB_LIBRARIES}) endif() add_executable(example_websocket websocket/example_ws.cpp) target_link_libraries(example_websocket ${Boost_LIBRARIES}) target_link_libraries(example_websocket ${CMAKE_THREAD_LIBS_INIT} ${OPENSSL_LIBRARIES}) +target_link_libraries(example_websocket ${ZLIB_LIBRARIES}) add_custom_command(OUTPUT ws.html COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/websocket/templates/ws.html ${CMAKE_CURRENT_BINARY_DIR}/templates/ws.html @@ -29,6 +43,7 @@ add_custom_target(example_ws_copy ALL DEPENDS ws.html) add_executable(basic_example example.cpp) target_link_libraries(basic_example ${Boost_LIBRARIES}) target_link_libraries(basic_example ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(basic_example ${ZLIB_LIBRARIES}) if (Tcmalloc_FOUND) target_link_libraries(basic_example ${Tcmalloc_LIBRARIES}) @@ -38,6 +53,7 @@ add_executable(example_with_all example_with_all.cpp) add_dependencies(example_with_all amalgamation) target_link_libraries(example_with_all ${Boost_LIBRARIES}) target_link_libraries(example_with_all ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(example_with_all ${ZLIB_LIBRARIES}) add_custom_command(OUTPUT example_test.py COMMAND ${CMAKE_COMMAND} -E @@ -49,6 +65,7 @@ add_custom_target(example_copy ALL DEPENDS example_test.py) add_executable(example_chat example_chat.cpp) target_link_libraries(example_chat ${Boost_LIBRARIES}) target_link_libraries(example_chat ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(example_chat ${ZLIB_LIBRARIES}) add_custom_command(OUTPUT example_chat.html COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/example_chat.html ${CMAKE_CURRENT_BINARY_DIR}/example_chat.html diff --git a/examples/example_compression.cpp b/examples/example_compression.cpp new file mode 100644 index 000000000..55e014a13 --- /dev/null +++ b/examples/example_compression.cpp @@ -0,0 +1,29 @@ +#include "crow.h" +#include "crow/compression.h" + +int main() +{ + crow::SimpleApp app; + //crow::App app; + + CROW_ROUTE(app, "/hello") + ([&](const crow::request&, crow::response& res){ + res.compressed = false; + + res.body = "Hello World! This is uncompressed!"; + res.end(); + }); + + CROW_ROUTE(app, "/hello_compressed") + ([](){ + return "Hello World! This is compressed by default!"; + }); + + + app.port(18080) + .use_compression(crow::compression::algorithm::DEFLATE) + //.use_compression(crow::compression::algorithm::GZIP) + .loglevel(crow::LogLevel::Debug) + .multithreaded() + .run(); +} diff --git a/include/crow.h b/include/crow.h index ba5633ee5..93ca96d1e 100644 --- a/include/crow.h +++ b/include/crow.h @@ -18,6 +18,7 @@ #include "crow/multipart.h" #include "crow/routing.h" #include "crow/middleware_context.h" +#include "crow/compression.h" #include "crow/http_connection.h" #include "crow/http_server.h" #include "crow/app.h" diff --git a/include/crow/app.h b/include/crow/app.h index 780a1b1e9..18b06fa59 100644 --- a/include/crow/app.h +++ b/include/crow/app.h @@ -18,6 +18,7 @@ #include "crow/http_request.h" #include "crow/http_server.h" #include "crow/dumb_timer_queue.h" +#include "crow/compression.h" #ifdef CROW_MSVC_WORKAROUND @@ -160,6 +161,18 @@ namespace crow return *this; } + self_t& use_compression(compression::algorithm algorithm) + { + comp_algorithm_ = algorithm; + return *this; + } + + + compression::algorithm compression_algorithm() + { + return comp_algorithm_; + } + ///A wrapper for `validate()` in the router /// @@ -338,6 +351,7 @@ namespace crow std::string server_name_ = "Crow/0.2"; std::string bindaddr_ = "0.0.0.0"; Router router_; + compression::algorithm comp_algorithm_; std::chrono::milliseconds tick_interval_; std::function tick_function_; diff --git a/include/crow/compression.h b/include/crow/compression.h new file mode 100644 index 000000000..e51186955 --- /dev/null +++ b/include/crow/compression.h @@ -0,0 +1,97 @@ +#pragma once + +#include +#include + +// http://zlib.net/manual.html + +namespace crow +{ + namespace compression + { + // Values used in the 'windowBits' parameter for deflateInit2. + enum algorithm + { + // 15 is the default value for deflate + DEFLATE = 15, + // windowBits can also be greater than 15 for optional gzip encoding. + // Add 16 to windowBits to write a simple gzip header and trailer around the compressed data instead of a zlib wrapper. + GZIP = 15|16, + }; + + std::string compress_string(std::string const & str, algorithm algo) + { + std::string compressed_str; + z_stream stream{}; + // Initialize with the default values + if (::deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, algo, 8, Z_DEFAULT_STRATEGY) == Z_OK) + { + char buffer[8192]; + + stream.avail_in = str.size(); + // zlib does not take a const pointer. The data is not altered. + stream.next_in = const_cast(reinterpret_cast(str.c_str())); + + int code = Z_OK; + do + { + stream.avail_out = sizeof(buffer); + stream.next_out = reinterpret_cast(&buffer[0]); + + code = ::deflate(&stream, Z_FINISH); + // Successful and non-fatal error code returned by deflate when used with Z_FINISH flush + if (code == Z_OK || code == Z_STREAM_END) + { + std::copy(&buffer[0], &buffer[sizeof(buffer) - stream.avail_out], std::back_inserter(compressed_str)); + } + + } while (code == Z_OK); + + if (code != Z_STREAM_END) + compressed_str.clear(); + + ::deflateEnd(&stream); + } + + return compressed_str; + } + + std::string decompress_string(std::string const & deflated_string) + { + std::string inflated_string; + Bytef tmp[8192]; + + z_stream zstream{}; + zstream.avail_in = deflated_string.size(); + // Nasty const_cast but zlib won't alter its contents + zstream.next_in = const_cast(reinterpret_cast(deflated_string.c_str())); + // Initialize with automatic header detection, for gzip support + if (::inflateInit2(&zstream, MAX_WBITS | 32) == Z_OK) + { + do + { + zstream.avail_out = sizeof(tmp); + zstream.next_out = &tmp[0]; + + auto ret = ::inflate(&zstream, Z_NO_FLUSH); + if (ret == Z_OK || ret == Z_STREAM_END) + { + std::copy(&tmp[0], &tmp[sizeof(tmp) - zstream.avail_out], std::back_inserter(inflated_string)); + } + else + { + // Something went wrong with inflate; make sure we return an empty string + inflated_string.clear(); + break; + } + + } while (zstream.avail_out == 0); + + // Free zlib's internal memory + ::inflateEnd(&zstream); + } + + return inflated_string; + } + } +} diff --git a/include/crow/http_connection.h b/include/crow/http_connection.h index cd3ec4c47..3f3cfe6f0 100644 --- a/include/crow/http_connection.h +++ b/include/crow/http_connection.h @@ -16,6 +16,7 @@ #include "crow/dumb_timer_queue.h" #include "crow/middleware_context.h" #include "crow/socket_adaptors.h" +#include "crow/compression.h" namespace crow { @@ -355,6 +356,32 @@ namespace crow decltype(*middlewares_)> (*middlewares_, ctx_, req_, res); } + + + std::string accept_encoding = req_.get_header_value("Accept-Encoding"); + if (!accept_encoding.empty() && res.compressed) + { + switch (handler_->compression_algorithm()) + { + case compression::DEFLATE: + if (accept_encoding.find("deflate") != std::string::npos) + { + res.body = compression::compress_string(res.body, compression::algorithm::DEFLATE); + res.set_header("Content-Encoding", "deflate"); + } + break; + case compression::GZIP: + if (accept_encoding.find("gzip") != std::string::npos) + { + res.body = compression::compress_string(res.body, compression::algorithm::GZIP); + res.set_header("Content-Encoding", "gzip"); + } + break; + default: + break; + } + } + prepare_buffers(); CROW_LOG_INFO << "Response: " << this << ' ' << req_.raw_url << ' ' << res.code << ' ' << close_connection_; if (res.is_static_type()) diff --git a/include/crow/http_response.h b/include/crow/http_response.h index e3fc39a38..78bf8bed8 100644 --- a/include/crow/http_response.h +++ b/include/crow/http_response.h @@ -30,6 +30,7 @@ namespace crow std::string body; ///< The actual payload containing the response data. json::wvalue json_value; ///< if the response body is JSON, this would be it. ci_map headers; ///< HTTP headers. + bool compressed = true; ///< If compression is enabled and this is false, the individual response will not be compressed. /// Set the value of an existing header in the response. void set_header(std::string key, std::string value) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7d78172b5..616a4126d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,7 @@ set(TEST_SRCS add_executable(unittest ${TEST_SRCS}) target_link_libraries(unittest ${Boost_LIBRARIES}) target_link_libraries(unittest ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(unittest ${ZLIB_LIBRARIES}) set_target_properties(unittest PROPERTIES COMPILE_FLAGS "-Wall -Werror -std=c++14") if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") diff --git a/tests/unittest.cpp b/tests/unittest.cpp index 32cb5a5d7..97283367a 100644 --- a/tests/unittest.cpp +++ b/tests/unittest.cpp @@ -1506,3 +1506,198 @@ TEST_CASE("websocket") app.stop(); } + +TEST_CASE("zlib_compression") +{ + static char buf_deflate[2048]; + static char buf_gzip[2048]; + + SimpleApp app_deflate, app_gzip; + + std::string expected_string = "Although moreover mistaken kindness me feelings do be marianne. Son over own nay with tell they cold upon are. " + "Cordial village and settled she ability law herself. Finished why bringing but sir bachelor unpacked any thoughts. " + "Unpleasing unsatiable particular inquietude did nor sir. Get his declared appetite distance his together now families. " + "Friends am himself at on norland it viewing. Suspected elsewhere you belonging continued commanded she."; + + // test deflate + CROW_ROUTE(app_deflate, "/test_compress")([&](){ + return expected_string; + }); + + CROW_ROUTE(app_deflate, "/test")([&](const request&, response& res){ + res.compressed = false; + + res.body = expected_string; + res.end(); + }); + + // test gzip + CROW_ROUTE(app_gzip, "/test_compress")([&](){ + return expected_string; + }); + + CROW_ROUTE(app_gzip, "/test")([&](const request&, response& res){ + res.compressed = false; + + res.body = expected_string; + res.end(); + }); + + auto t1 = async(launch::async, [&]{ app_deflate.bindaddr(LOCALHOST_ADDRESS).port(45451).use_compression(compression::algorithm::DEFLATE).run(); }); + auto t2 = async(launch::async, [&]{ app_gzip.bindaddr(LOCALHOST_ADDRESS).port(45452).use_compression(compression::algorithm::GZIP).run(); }); + + app_deflate.wait_for_server_start(); + app_gzip.wait_for_server_start(); + + std::string test_compress_msg = "GET /test_compress\r\nAccept-Encoding: gzip, deflate\r\n\r\n"; + std::string test_compress_no_header_msg = "GET /test_compress\r\n\r\n"; + std::string test_none_msg = "GET /test\r\n\r\n"; + + auto inflate_string = [](std::string const & deflated_string) -> const std::string + { + std::string inflated_string; + Bytef tmp[8192]; + + z_stream zstream{}; + zstream.avail_in = deflated_string.size(); + // Nasty const_cast but zlib won't alter its contents + zstream.next_in = const_cast(reinterpret_cast(deflated_string.c_str())); + // Initialize with automatic header detection, for gzip support + if (::inflateInit2(&zstream, MAX_WBITS | 32) == Z_OK) + { + do + { + zstream.avail_out = sizeof(tmp); + zstream.next_out = &tmp[0]; + + auto ret = ::inflate(&zstream, Z_NO_FLUSH); + if (ret == Z_OK || ret == Z_STREAM_END) + { + std::copy(&tmp[0], &tmp[sizeof(tmp) - zstream.avail_out], std::back_inserter(inflated_string)); + } + else + { + // Something went wrong with inflate; make sure we return an empty string + inflated_string.clear(); + break; + } + + } while (zstream.avail_out == 0); + + // Free zlib's internal memory + ::inflateEnd(&zstream); + } + + return inflated_string; + }; + + std::string response_deflate; + std::string response_gzip; + std::string response_deflate_no_header; + std::string response_gzip_no_header; + std::string response_deflate_none; + std::string response_gzip_none; + + auto on_body = [](http_parser * parser, const char * body, size_t body_length) -> int + { + std::string * body_ptr = reinterpret_cast(parser->data); + *body_ptr = std::string(body, body + body_length); + + return 0; + }; + + http_parser_settings settings{}; + settings.on_body = on_body; + + asio::io_service is; + { + // Compression + { + asio::ip::tcp::socket socket[2] = { asio::ip::tcp::socket(is), asio::ip::tcp::socket(is) }; + socket[0].connect(asio::ip::tcp::endpoint(asio::ip::address::from_string(LOCALHOST_ADDRESS), 45451)); + socket[1].connect(asio::ip::tcp::endpoint(asio::ip::address::from_string(LOCALHOST_ADDRESS), 45452)); + + socket[0].send(asio::buffer(test_compress_msg)); + socket[1].send(asio::buffer(test_compress_msg)); + + size_t bytes_deflate = socket[0].receive(asio::buffer(buf_deflate, 2048)); + size_t bytes_gzip = socket[1].receive(asio::buffer(buf_gzip, 2048)); + + http_parser parser[2] = { {}, {} }; + http_parser_init(&parser[0], HTTP_RESPONSE); + http_parser_init(&parser[1], HTTP_RESPONSE); + parser[0].data = reinterpret_cast(&response_deflate); + parser[1].data = reinterpret_cast(&response_gzip); + + http_parser_execute(&parser[0], &settings, buf_deflate, bytes_deflate); + http_parser_execute(&parser[1], &settings, buf_gzip, bytes_gzip); + + response_deflate = inflate_string(response_deflate); + response_gzip = inflate_string(response_gzip); + + socket[0].close(); + socket[1].close(); + } + // No Header (thus no compression) + { + asio::ip::tcp::socket socket[2] = { asio::ip::tcp::socket(is), asio::ip::tcp::socket(is) }; + socket[0].connect(asio::ip::tcp::endpoint(asio::ip::address::from_string(LOCALHOST_ADDRESS), 45451)); + socket[1].connect(asio::ip::tcp::endpoint(asio::ip::address::from_string(LOCALHOST_ADDRESS), 45452)); + + socket[0].send(asio::buffer(test_compress_no_header_msg)); + socket[1].send(asio::buffer(test_compress_no_header_msg)); + + size_t bytes_deflate = socket[0].receive(asio::buffer(buf_deflate, 2048)); + size_t bytes_gzip = socket[1].receive(asio::buffer(buf_gzip, 2048)); + + http_parser parser[2] = { {}, {} }; + http_parser_init(&parser[0], HTTP_RESPONSE); + http_parser_init(&parser[1], HTTP_RESPONSE); + parser[0].data = reinterpret_cast(&response_deflate_no_header); + parser[1].data = reinterpret_cast(&response_gzip_no_header); + + http_parser_execute(&parser[0], &settings, buf_deflate, bytes_deflate); + http_parser_execute(&parser[1], &settings, buf_gzip, bytes_gzip); + + socket[0].close(); + socket[1].close(); + } + // No compression + { + asio::ip::tcp::socket socket[2] = { asio::ip::tcp::socket(is), asio::ip::tcp::socket(is) }; + socket[0].connect(asio::ip::tcp::endpoint(asio::ip::address::from_string(LOCALHOST_ADDRESS), 45451)); + socket[1].connect(asio::ip::tcp::endpoint(asio::ip::address::from_string(LOCALHOST_ADDRESS), 45452)); + + socket[0].send(asio::buffer(test_none_msg)); + socket[1].send(asio::buffer(test_none_msg)); + + size_t bytes_deflate = socket[0].receive(asio::buffer(buf_deflate, 2048)); + size_t bytes_gzip = socket[1].receive(asio::buffer(buf_gzip, 2048)); + + http_parser parser[2] = { {}, {} }; + http_parser_init(&parser[0], HTTP_RESPONSE); + http_parser_init(&parser[1], HTTP_RESPONSE); + parser[0].data = reinterpret_cast(&response_deflate_none); + parser[1].data = reinterpret_cast(&response_gzip_none); + + http_parser_execute(&parser[0], &settings, buf_deflate, bytes_deflate); + http_parser_execute(&parser[1], &settings, buf_gzip, bytes_gzip); + + socket[0].close(); + socket[1].close(); + } + } + { + CHECK(expected_string == response_deflate); + CHECK(expected_string == response_gzip); + + CHECK(expected_string == response_deflate_no_header); + CHECK(expected_string == response_gzip_no_header); + + CHECK(expected_string == response_deflate_none); + CHECK(expected_string == response_gzip_none); + } + + app_deflate.stop(); + app_gzip.stop(); +}