From e9bd766b158be33b5616c385fefe51d806ba8d99 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 26 Dec 2022 16:46:17 +0100 Subject: [PATCH 1/8] Move default timeout to netlib namespace Add some preparations for http addition Add uri-parser as submodule --- .gitmodules | 3 +++ examples/http_client.cpp | 3 +++ extern/cpp-uri-parser | 1 + src/client.hpp | 3 +-- src/http/client.hpp | 8 ++++++++ src/http/http.hpp | 8 ++++++++ tests/test_http_parser.cpp | 3 +++ 7 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 examples/http_client.cpp create mode 160000 extern/cpp-uri-parser create mode 100644 src/http/client.hpp create mode 100644 src/http/http.hpp create mode 100644 tests/test_http_parser.cpp diff --git a/.gitmodules b/.gitmodules index 2cc23bd..a93d503 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "doctest"] path = doctest url = https://github.com/onqtam/doctest.git +[submodule "extern/cpp-uri-parser"] + path = extern/cpp-uri-parser + url = https://github.com/lpcvoid/cpp-uri-parser diff --git a/examples/http_client.cpp b/examples/http_client.cpp new file mode 100644 index 0000000..b7d8386 --- /dev/null +++ b/examples/http_client.cpp @@ -0,0 +1,3 @@ +// +// Created by lpcvoid on 25.12.22. +// diff --git a/extern/cpp-uri-parser b/extern/cpp-uri-parser new file mode 160000 index 0000000..b6ff039 --- /dev/null +++ b/extern/cpp-uri-parser @@ -0,0 +1 @@ +Subproject commit b6ff039ed36f2352fc8537b24971a6437faacdec diff --git a/src/client.hpp b/src/client.hpp index 94a8237..1965a52 100644 --- a/src/client.hpp +++ b/src/client.hpp @@ -13,10 +13,9 @@ namespace netlib { using namespace std::chrono_literals; +static constexpr std::chrono::milliseconds DEFAULT_TIMEOUT = 1000ms; class client { -private: - static constexpr std::chrono::milliseconds DEFAULT_TIMEOUT = 1000ms; protected: std::optional _socket; addrinfo *_endpoint_addr = nullptr; diff --git a/src/http/client.hpp b/src/http/client.hpp new file mode 100644 index 0000000..77d10c0 --- /dev/null +++ b/src/http/client.hpp @@ -0,0 +1,8 @@ +// +// Created by lpcvoid on 25.12.22. +// + +#ifndef NETLIB_CLIENT_HPP +#define NETLIB_CLIENT_HPP + +#endif // NETLIB_CLIENT_HPP diff --git a/src/http/http.hpp b/src/http/http.hpp new file mode 100644 index 0000000..4d13857 --- /dev/null +++ b/src/http/http.hpp @@ -0,0 +1,8 @@ +// +// Created by lpcvoid on 25.12.22. +// + +#ifndef NETLIB_HTTP_HPP +#define NETLIB_HTTP_HPP + +#endif // NETLIB_HTTP_HPP diff --git a/tests/test_http_parser.cpp b/tests/test_http_parser.cpp new file mode 100644 index 0000000..95a01d8 --- /dev/null +++ b/tests/test_http_parser.cpp @@ -0,0 +1,3 @@ +// +// Created by lpcvoid on 26.12.22. +// From c6c7ab52172d8703aaaf4f9796f567de79b22180 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 26 Dec 2022 16:50:38 +0100 Subject: [PATCH 2/8] Add rudimentary http support (client only so far) --- CMakeLists.txt | 18 ++++++++- src/http/client.hpp | 87 +++++++++++++++++++++++++++++++++++++++--- src/http/http.hpp | 92 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 184 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 729dea3..732af44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,13 +19,29 @@ set(NETLIB_SRC src/service_resolver.hpp src/endpoint_accessor.hpp src/thread_pool.hpp - src/socket_operations.hpp) + src/socket_operations.hpp +) + +set(NETLIB_HTTP + src/http/client.hpp + src/http/http.hpp +) option(BUILD_TESTS "Build tests" ON) option(BUILD_EXAMPLES "Build example programs" ON) +option(WITH_HTTP "Build with http support" ON) if(NOT EXISTS "${CMAKE_SOURCE_DIR}/doctest/CMakeLists.txt") set(BUILD_TESTS OFF) + message(WARNING "Cannot build tests without doctest! Deactivating tests.") +endif() + +if (WITH_HTTP) + if(NOT EXISTS "${CMAKE_SOURCE_DIR}/extern/cpp-uri-parser/URI.hpp") + set(WITH_HTTP OFF) + message(WARNING "If you want HTTP support, you need to check out submodules (cpp-uri-parser)") + endif() + message(NOTICE "Building with HTTP support.") endif() if(BUILD_TESTS) diff --git a/src/http/client.hpp b/src/http/client.hpp index 77d10c0..91d0971 100644 --- a/src/http/client.hpp +++ b/src/http/client.hpp @@ -1,8 +1,83 @@ -// -// Created by lpcvoid on 25.12.22. -// +#include "../extern/cpp-uri-parser/URI.hpp" +#include "http.hpp" +#pragma once -#ifndef NETLIB_CLIENT_HPP -#define NETLIB_CLIENT_HPP +namespace netlib::http { -#endif // NETLIB_CLIENT_HPP +using namespace std::chrono_literals; + +class http_client { +private: + netlib::thread_pool _thread_pool; + netlib::client _client; +public: + + inline http_client() {} + + inline std::pair, std::error_condition> get(const std::string& url) { + auto uri = URI(url); + if (uri.get_result() != URI::URIParsingResult::success) { + return std::make_pair(std::nullopt, std::errc::bad_address); + } + if (!_client.is_connected()) { + uint16_t port = 80; + if (uri.get_protocol().has_value()) { + std::string_view protocol_str = uri.get_protocol().value(); + if (protocol_str != "http"){ + // no support for TLS so far... + // also, only http and https shall ever be expected + return std::make_pair(std::nullopt, std::errc::protocol_not_supported); + } + } + auto res = _client.connect(std::string(uri.get_host().value()), port, AddressFamily::IPv4, AddressProtocol::TCP); + if (res) { + return std::make_pair(std::nullopt, res); + } + } + + std::string query = (uri.get_query().has_value() ? std::string(uri.get_query().value()) : "/"); + std::string http_get = "GET " + query + " HTTP/1.1\r\nHost:" + std::string(uri.get_host().value()) + "\r\n\r\n"; + const std::vector get_data(http_get.begin(), http_get.end()); + auto send_res = _client.send(get_data); + if (send_res.first != get_data.size()) { + return std::make_pair(std::nullopt, send_res.second); + } + std::vector data_buffer; + std::chrono::milliseconds time_spent = std::chrono::milliseconds(0); + const std::chrono::milliseconds TICK_TIME = std::chrono::milliseconds(50); + while (true) { + auto recv_res = _client.recv(0, TICK_TIME); + time_spent += TICK_TIME; + if (!recv_res.first.empty()) { + data_buffer.insert(data_buffer.begin(), recv_res.first.begin(), recv_res.first.end()); + } + if ((recv_res.second == std::errc::timed_out) && (!data_buffer.empty())) { + netlib::http::http_response response; + std::string raw_response (data_buffer.begin(), data_buffer.end()); + std::error_condition parse_resp = response.from_raw_response(raw_response); + if (!parse_resp) { + return {response, {}}; + } else { + return {std::nullopt, parse_resp}; + } + } + if (time_spent > DEFAULT_TIMEOUT) { + return {std::nullopt, std::errc::timed_out}; + } + } + } + + inline std::future, std::error_condition>> get_async(const std::string& url) { + return _thread_pool.add_task( + [&](std::string url) { + return this->get(url); + }, + url); + } + + + + +}; + +} diff --git a/src/http/http.hpp b/src/http/http.hpp index 4d13857..91402ee 100644 --- a/src/http/http.hpp +++ b/src/http/http.hpp @@ -1,8 +1,88 @@ -// -// Created by lpcvoid on 25.12.22. -// +#pragma once -#ifndef NETLIB_HTTP_HPP -#define NETLIB_HTTP_HPP +#include +#include +#include +#include +namespace netlib::http { -#endif // NETLIB_HTTP_HPP +using http_header_entry = std::pair; +using http_headers = std::vector; + +struct http_response { + http_headers headers; + uint32_t response_code; + std::pair version; + std::string body; + inline std::error_condition from_raw_response(const std::string& raw_response) { + // a very rudimentary http response parser + if (raw_response.empty()) { + return std::errc::no_message; + } + + /* strategy: + * split into multiple (at least two) parts, delimited by \r\n\r\n" + * within the first part: + * first, parse the status line + * second, parse the header fields, until we arrive at an empty line (only CR LF) + * last, an optional body + * then, for the rest of the parts, concat into body + */ + + auto split = [](const std::string& str, const std::string& delimiter) -> std::vector { + std::vector split_tokens; + std::size_t start; + std::size_t end = 0; + while ((start = str.find_first_not_of(delimiter, end)) != std::string::npos) + { + end = str.find(delimiter, start); + split_tokens.push_back(str.substr(start, end - start)); + } + return split_tokens; + }; + + std::vector header_body_split = split(raw_response, "\r\n\r\n"); + //split header part of response into response_header_lines + std::vector response_header_lines = split(header_body_split.front(), "\r\n"); + //first line should start with "HTTP" + if (!response_header_lines.front().starts_with("HTTP")) { + return std::errc::result_out_of_range; + } + //attempt to parse status line + //split into parts by space + auto status_parts = split(response_header_lines.front(), " "); + if (status_parts.size() < 3) { + return std::errc::bad_message; + } + //parse "HTTP/x.x" + auto version_parts = split(status_parts.front(), "/"); + if (version_parts.size() != 2) { + return std::errc::bad_message; + } + //parse "x.x" + auto version_components = split(version_parts.back(), "."); + version.first = std::stoi(version_components.front()); + version.second = std::stoi(version_components.back()); + //parse response code + response_code = std::stoi(status_parts[1]); + //there can be an optional code description in the first line, but we ignore that here + //parse the response header lines until the end + //start at second line, first is status + std::for_each(response_header_lines.begin(), response_header_lines.end(), [&](const std::string& header_component){ + auto component_parts = split(header_component, ":"); + if (component_parts.size() == 2) { + headers.emplace_back(component_parts.front(), component_parts.back()); + } + }); + + //now, take the body part(s) and concat them + std::for_each(header_body_split.begin() + 1, header_body_split.end(), [&](const std::string& body_line){ + body += body_line; + }); + + return {}; + + }; +}; + +} \ No newline at end of file From dfd93c5dd08d7c87332c72b84257c89812cdddf5 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 26 Dec 2022 16:50:58 +0100 Subject: [PATCH 3/8] Add basic example for the http client --- examples/CMakeLists.txt | 4 ++++ examples/http_client.cpp | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index df02d36..311fc26 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,5 +1,9 @@ set(EXAMPLE_SOURCES echo_server.cpp daytime_client.cpp threadpool.cpp time_client.cpp) +if (WITH_HTTP) + set(EXAMPLE_SOURCES ${EXAMPLE_SOURCES} http_client.cpp) +endif() + foreach (examplesource ${EXAMPLE_SOURCES}) string(REPLACE ".cpp" "" examplename ${examplesource}) add_executable(${examplename} ${examplesource}) diff --git a/examples/http_client.cpp b/examples/http_client.cpp index b7d8386..7318634 100644 --- a/examples/http_client.cpp +++ b/examples/http_client.cpp @@ -1,3 +1,35 @@ -// -// Created by lpcvoid on 25.12.22. -// +#include "../src/netlib.hpp" +#include "../src/http/client.hpp" +#include "../src/http/http.hpp" +#include +#include +#include + + +void exit_handler(int s){ + std::cout << "Goodbye!" << std::endl; + exit(EXIT_SUCCESS); +} + +int main(int argc, char** argv) +{ + netlib::http::http_client client; + + auto res = client.get("http://example.com"); + + if (res.second) { + std::cerr << "Error: " << res.second.message() << std::endl; + exit(1); + } + + std::cout << "Got HTTP response: " << res.first->response_code << + ", version " << res.first->version.first << "." << res.first->version.second << std::endl; + std::cout << "Header entries:" << std::endl; + std::for_each(res.first->headers.begin(), res.first->headers.end(), [](auto header_entry) { + std::cout << header_entry.first << " = " << header_entry.second << std::endl; + }); + std::cout << "Body:" << std::endl; + std::cout << res.first.value().body << std::endl; + + signal(SIGINT, exit_handler); +} \ No newline at end of file From 3df66db7987a1d8106821e93f869c530b6ae3c64 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 26 Dec 2022 16:51:22 +0100 Subject: [PATCH 4/8] Add http parsing test --- tests/CMakeLists.txt | 4 ++++ tests/test_http_parser.cpp | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a0791d1..637882c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,6 +5,10 @@ set(TEST_SOURCES test_large_data_transfer.cpp test_raii.cpp) +if (WITH_HTTP) + set(TEST_SOURCES ${TEST_SOURCES} test_http_parser.cpp) +endif() + enable_testing() add_executable(test_libnetcpp main.cpp ${TEST_SOURCES}) target_link_libraries(test_libnetcpp ${CMAKE_THREAD_LIBS_INIT}) diff --git a/tests/test_http_parser.cpp b/tests/test_http_parser.cpp index 95a01d8..f796637 100644 --- a/tests/test_http_parser.cpp +++ b/tests/test_http_parser.cpp @@ -1,3 +1,30 @@ -// -// Created by lpcvoid on 26.12.22. -// +#include "../doctest/doctest/doctest.h" +#include "../src/http/http.hpp" +#include "../src/netlib.hpp" +#include + +using namespace std::chrono_literals; +extern uint16_t test_port; + +TEST_CASE("Test HTTP response parser") +{ + netlib::http::http_response resp; + + std::string raw_response = "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html; charset=UTF-8\r\n" + "Content-Length: 1256\r\n" + "\r\n\r\n" + "TEST"; + + CHECK_FALSE(resp.from_raw_response(raw_response)); + CHECK_EQ(resp.version.first, 1); + CHECK_EQ(resp.version.second, 1); + CHECK_EQ(resp.response_code, 200); + CHECK_EQ(resp.headers.size(), 2); + CHECK_EQ(resp.headers[0].first, "Content-Type"); + CHECK_EQ(resp.headers[1].first, "Content-Length"); + CHECK_EQ(resp.headers[0].second, " text/html; charset=UTF-8"); + CHECK_EQ(resp.headers[1].second, " 1256"); + CHECK_EQ(resp.body, "TEST"); + +} \ No newline at end of file From ed8cc3d63abdc49629249e210982b0b8e7a6ee21 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 26 Dec 2022 17:03:12 +0100 Subject: [PATCH 5/8] Fix macos compilation warnings --- src/socket.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/socket.hpp b/src/socket.hpp index 4cbb1cf..d51686f 100644 --- a/src/socket.hpp +++ b/src/socket.hpp @@ -42,7 +42,9 @@ using socket_t = int32_t; // this is actually a nice one to have #define INVALID_SOCKET (-1) #ifdef __APPLE__ -#define MSG_NOSIGNAL SO_NOSIGPIPE + #ifndef MSG_NOSIGNAL + #define MSG_NOSIGNAL SO_NOSIGPIPE + #endif #endif #endif From 90bb8218105ae097258083cb1f50a5ab6e4a7abc Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 26 Dec 2022 17:09:02 +0100 Subject: [PATCH 6/8] updated test matrix --- .github/workflows/test_matrix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_matrix.yml b/.github/workflows/test_matrix.yml index 4c38db1..286e7b1 100644 --- a/.github/workflows/test_matrix.yml +++ b/.github/workflows/test_matrix.yml @@ -18,7 +18,7 @@ jobs: os: windows-latest, build_type: "Release", cc: "cl", cxx: "cl", #this path will change at some point when VC2022 is released I guess - environment_script: "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Auxiliary/Build/vcvars64.bat" + environment_script: "C:/Program Files (x86)/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvars64.bat" } - { name: "Ubuntu gcc", @@ -31,9 +31,9 @@ jobs: build_type: "Release", cc: "clang", cxx: "clang++" } - { - name: "MacOS clang", + name: "MacOS gcc", os: macos-latest, - build_type: "Release", cc: "clang", cxx: "clang++" + build_type: "Release", cc: "gcc", cxx: "g++" } steps: - uses: actions/checkout@v2 From 827396445e8c7a57a415ef56e8809081b16671b9 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 26 Dec 2022 17:29:55 +0100 Subject: [PATCH 7/8] fixed windows compile warnings --- src/http/http.hpp | 2 ++ src/socket.hpp | 8 ++++++++ src/socket_operations.hpp | 2 +- tests/test_http_parser.cpp | 1 - 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/http/http.hpp b/src/http/http.hpp index 91402ee..457d4ab 100644 --- a/src/http/http.hpp +++ b/src/http/http.hpp @@ -4,6 +4,8 @@ #include #include #include +#include + namespace netlib::http { using http_header_entry = std::pair; diff --git a/src/socket.hpp b/src/socket.hpp index d51686f..287d8f3 100644 --- a/src/socket.hpp +++ b/src/socket.hpp @@ -25,6 +25,14 @@ using ssize_t = signed long long int; #define MSG_NOSIGNAL 0 //poll implementation #define poll_syscall ::WSAPoll +//we ignore unused parameter warning on windows, missing impls +#pragma warning(disable: 4100) +//use unsafe functions under windows +#define _CRT_SECURE_NO_WARNINGS +#pragma warning(disable:4996) +#pragma warning(disable:4267) +#pragma warning(disable:4244) + #else // headers #include diff --git a/src/socket_operations.hpp b/src/socket_operations.hpp index 7d504c1..1f37e53 100644 --- a/src/socket_operations.hpp +++ b/src/socket_operations.hpp @@ -29,7 +29,7 @@ class operations { std::size_t remaining_bytes = data.size() - total_sent_size; std::size_t actual_chunk_size = (remaining_bytes > chunk_size) ? chunk_size : remaining_bytes; - ssize_t send_res = ::send(sock.get_raw().value(), data_pointer, actual_chunk_size, MSG_NOSIGNAL); + ssize_t send_res = ::send(sock.get_raw().value(), data_pointer, static_cast(actual_chunk_size), MSG_NOSIGNAL); if (send_res > 0) { total_sent_size += static_cast(send_res); data_pointer += send_res; diff --git a/tests/test_http_parser.cpp b/tests/test_http_parser.cpp index f796637..3afa000 100644 --- a/tests/test_http_parser.cpp +++ b/tests/test_http_parser.cpp @@ -1,7 +1,6 @@ #include "../doctest/doctest/doctest.h" #include "../src/http/http.hpp" #include "../src/netlib.hpp" -#include using namespace std::chrono_literals; extern uint16_t test_port; From e9a37e523bbe6061fe7508d52682a1e510570ffe Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 26 Dec 2022 22:04:33 +0100 Subject: [PATCH 8/8] update submodule --- extern/cpp-uri-parser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/cpp-uri-parser b/extern/cpp-uri-parser index b6ff039..ed011f5 160000 --- a/extern/cpp-uri-parser +++ b/extern/cpp-uri-parser @@ -1 +1 @@ -Subproject commit b6ff039ed36f2352fc8537b24971a6437faacdec +Subproject commit ed011f504c72ddcddd92893e9612939eeaadd521