From 204bb3d33a1a2cb509ae36c7ecfe977f7d9cfe86 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Mon, 5 Jun 2017 03:58:54 +0700 Subject: [PATCH 01/26] Initial system preparation for Libmicrohttpd integration * .gitlab-ci.yml: Add Libmicrohttpd package for CI testing using Debian/gcc, Fedora/clang and MinGW64 * .travis.sh: Add Libmicrohttpd package for OSX/clang * .travis_setup.sh: Add Libmicrohttpd package for Debian/gcc and clang * README.md: Add info which mentioning Libmicrohttpd as optional requirement * configure.ac: Check for Libmicrohttpd availability --- .gitlab-ci.yml | 23 ++++++++++++++++++++--- .travis.sh | 9 +++++++++ .travis_setup.sh | 6 ++++++ README.md | 1 + configure.ac | 15 +++++++++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69fd09c30..36733c8db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,8 @@ variables: # * ASan, UBSan, Msan # * distcheck Debian GNU/Linux build: + before_script: + - apt-get update -qq && apt-get install -y -qq libmicrohttpd-dev image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD script: - ./bootstrap && touch .manywarnings @@ -56,6 +58,8 @@ Debian GNU/Linux build: - tests/*.log clang/Fedora: + before_script: + - dnf -y install libmicrohttpd-devel image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - export CC=clang @@ -117,15 +121,28 @@ pages: MinGW64: image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD script: - - apt-get -y install gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-x86-64-dev pkg-config-mingw-w64-x86-64 win-iconv-mingw-w64-dev wine - - ./bootstrap + - dpkg --add-architecture i386 + - apt-get -y update + - apt-get -y install gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-x86-64-dev pkg-config-mingw-w64-x86-64 win-iconv-mingw-w64-dev wine wine32 wget + - cd .. - export PREFIX=x86_64-w64-mingw32 - export CC=$PREFIX-gcc - - export CXX=$PREFIX-g++ - export CPP=$PREFIX-cpp - export RANLIB=$PREFIX-ranlib - export PATH="/usr/$PREFIX/bin:$PATH" - export CFLAGS="-O2 -Wall -Wno-format" + # Install Libmicrohttpd from source + - git clone https://gnunet.org/git/libmicrohttpd.git + - cd libmicrohttpd + - git checkout c0168fde5043523 + - ./bootstrap + - ./configure --build=x86_64-pc-linux-gnu --host=$PREFIX --prefix=/usr/$PREFIX --disable-doc --disable-examples --enable-shared --enable-static + - make clean + - make -j$(nproc) + - make install + - cd - && cd wget2 + - ./bootstrap + - export CXX=$PREFIX-g++ - export WINEPATH="/usr/$PREFIX/bin;/usr/$PREFIX/lib;$PWD/libwget/.libs" - ./configure --build=x86_64-pc-linux-gnu --host=$PREFIX --enable-shared --enable-static - make clean diff --git a/.travis.sh b/.travis.sh index 95558cffa..af857ee0d 100755 --- a/.travis.sh +++ b/.travis.sh @@ -10,6 +10,15 @@ test "$CC" = "clang" && export CXX="clang++" if [[ $TRAVIS_OS_NAME = 'osx' ]]; then CONFIGURE_OPTIONS+=("") + # Install Libmicrohttpd from source + cd .. + wget http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.55.tar.gz + tar zxf libmicrohttpd-0.9.55.tar.gz && cd libmicrohttpd-0.9.55 + ./configure + make clean + make -j3 + make install + cd - && cd wget2 else CONFIGURE_OPTIONS+=("--enable-valgrind-tests") fi diff --git a/.travis_setup.sh b/.travis_setup.sh index 916dbdb7e..386a15674 100755 --- a/.travis_setup.sh +++ b/.travis_setup.sh @@ -14,7 +14,13 @@ if [[ "$TRAVIS_OS_NAME" = "osx" ]]; then brew install xz brew install lbzip2 brew install lzip + brew install libgcrypt brew link --force gettext elif [[ "$TRAVIS_OS_NAME" = "linux" ]]; then + # Install Libmicrohttpd from source + sudo apt-get -y install wget + wget http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.55.tar.gz + tar zxf libmicrohttpd-0.9.55.tar.gz && cd libmicrohttpd-0.9.55/ + ./configure --prefix=/usr && make -j$(nproc) && sudo make install pip install --user cpp-coveralls fi diff --git a/README.md b/README.md index c5447abe9..87faf2ba4 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ The following packages are needed to build the software * flex >= 2.5.35 * libpsl >= 0.5.0 * libnghttp2 >= 1.3.0 (optional, if you want HTTP/2 support) +* libmicrohttpd >= 0.9.51 (optional, if you want to run the test suite) The versions are recommended, but older versions may also work. diff --git a/configure.ac b/configure.ac index edda77b63..2a9a4adb9 100644 --- a/configure.ac +++ b/configure.ac @@ -546,6 +546,20 @@ AS_IF([test "x$with_libidn2" != xyes], [ ]) AM_CONDITIONAL([WITH_LIBIDN], [test "x$with_libidn" = xyes]) +AC_ARG_WITH(libmicrohttpd, AS_HELP_STRING([--without-libmicrohttpd], [disable support for libmicrohttpd]), with_libmicrohttpd=$withval, with_libmicrohttpd=yes) +AS_IF([test "x$with_libmicrohttpd" != xno], [ + PKG_CHECK_MODULES(MICROHTTPD, [libmicrohttpd], [ + with_libmicrohttpd=yes + LIBS="$MICROHTTPD_LIBS $LIBS" + CFLAGS="$MICROHTTPD_CFLAGS $CFLAGS" + AC_DEFINE([WITH_MICROHTTPD], [1], [Use Libmicrohttpd]) + ], [ + with_libmicrohttpd=no + AC_MSG_WARN(*** GNU Libmicrohttpd was not found. You will not be able to run the test suite.)] + ) +]) +AM_CONDITIONAL([WITH_MICROHTTPD], [test "x$with_libmicrohttpd" = xyes]) + # Check for plugin support AC_CANONICAL_HOST case "$host_os" in @@ -628,4 +642,5 @@ AC_MSG_NOTICE([Summary of build options: Tests: ${TESTS_INFO} Assertions: $ENABLE_ASSERT POSIX xattr: $ENABLE_XATTR + Microhttpd support: $with_libmicrohttpd ]) From 06b1ee35375fc9db022e67d402899ad7ef3d82ca Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Tue, 13 Jun 2017 10:46:25 +0700 Subject: [PATCH 02/26] * tests/libtest.c: Create Libmicrohttpd function wrapper --- tests/libtest.c | 84 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/tests/libtest.c b/tests/libtest.c index ea19f89b7..b0a0f08c6 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -45,8 +45,11 @@ #include #include "libtest.h" +#ifdef WITH_MICROHTTPD + #include +#endif + static wget_thread_t - http_server_tid, https_server_tid, ftp_server_tid, ftps_server_tid; @@ -83,6 +86,12 @@ static const char static char server_send_content_length = 1; +#ifdef WITH_MICROHTTPD +// MHD_Daemon instance +struct MHD_Daemon + *httpdaemon; +#endif + static void sigterm_handler(int sig G_GNUC_WGET_UNUSED) { terminate = 1; @@ -257,6 +266,55 @@ static void *_http_server_thread(void *ctx) return NULL; } +#ifdef WITH_MICROHTTPD +static int _answer_to_connection(void *cls, + struct MHD_Connection *connection, + const char *url, + const char *method, + const char *version, + const char *upload_data, size_t *upload_data_size, void **con_cls) +{ + struct MHD_Response *response; + int ret; + + // it1 = iteration for urls data + unsigned int it1, found = 0; + for (it1 = 0; it1 < nurls; it1++) { + if (!strcmp(url, urls[it1].name)) + { + response = MHD_create_response_from_buffer(strlen(urls[it1].body), + (void *) urls[it1].body, MHD_RESPMEM_MUST_COPY); + ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + + it1 = nurls; + found = 1; + } + } + + MHD_destroy_response(response); + return ret; +} + +static void _http_server_stop(void) +{ + MHD_stop_daemon(httpdaemon); +} + +static int _http_server_start(void) +{ + int port_num = 0; + + httpdaemon = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, + port_num, NULL, NULL, &_answer_to_connection, NULL, NULL, + MHD_OPTION_END); + + if (!httpdaemon) + return 1; + + return 0; +} +#endif + static void *_ftp_server_thread(void *ctx) { wget_tcp_t *tcp = NULL, *parent_tcp = ctx, *pasv_parent_tcp = NULL, *pasv_tcp = NULL; @@ -472,7 +530,6 @@ void wget_test_stop_server(void) // if (ftps_implicit) // pthread_kill(ftps_server_tid, SIGTERM); - wget_thread_cancel(http_server_tid); wget_thread_cancel(https_server_tid); wget_thread_cancel(ftp_server_tid); if (ftps_implicit) @@ -490,6 +547,9 @@ void wget_test_stop_server(void) _remove_directory(tmpdir); wget_global_deinit(); +#ifdef WITH_MICROHTTPD + _http_server_stop(); +#endif } static char *_insert_ports(const char *src) @@ -548,7 +608,7 @@ static void _write_msg(const char *msg, size_t len) void wget_test_start_server(int first_key, ...) { - static wget_tcp_t *http_parent_tcp, *https_parent_tcp, *ftp_parent_tcp, *ftps_parent_tcp; + static wget_tcp_t *https_parent_tcp, *ftp_parent_tcp, *ftps_parent_tcp; int rc, key; size_t it; va_list args; @@ -628,14 +688,6 @@ void wget_test_start_server(int first_key, ...) wget_ssl_set_config_string(WGET_SSL_CERT_FILE, SRCDIR "/certs/x509-server-cert.pem"); wget_ssl_set_config_string(WGET_SSL_KEY_FILE, SRCDIR "/certs/x509-server-key.pem"); - // init HTTP server socket - http_parent_tcp = wget_tcp_init(); - wget_tcp_set_timeout(http_parent_tcp, -1); // INFINITE timeout - wget_tcp_set_preferred_family(http_parent_tcp, WGET_NET_FAMILY_IPV4); // to have a defined order of IPs - if (wget_tcp_listen(http_parent_tcp, "localhost", 0, 5) != 0) - exit(1); - http_server_port = wget_tcp_get_local_port(http_parent_tcp); - // init HTTPS server socket https_parent_tcp = wget_tcp_init(); wget_tcp_set_ssl(https_parent_tcp, 1); // switch SSL on @@ -664,6 +716,12 @@ void wget_test_start_server(int first_key, ...) ftps_server_port = wget_tcp_get_local_port(ftps_parent_tcp); } +#ifdef WITH_MICROHTTPD + // start HTTP server + if ((rc = _http_server_start()) != 0) + wget_error_printf_exit(_("Failed to start HTTP server, error %d\n"), rc); +#endif + // now replace {{port}} in the body by the actual server port for (wget_test_url_t *url = urls; url < urls + nurls; url++) { char *p = _insert_ports(url->body); @@ -683,10 +741,6 @@ void wget_test_start_server(int first_key, ...) } } - // start thread for HTTP - if ((rc = wget_thread_start(&http_server_tid, _http_server_thread, http_parent_tcp, 0)) != 0) - wget_error_printf_exit(_("Failed to start HTTP server, error %d\n"), rc); - // start thread for HTTPS if ((rc = wget_thread_start(&https_server_tid, _http_server_thread, https_parent_tcp, 0)) != 0) wget_error_printf_exit(_("Failed to start HTTPS server, error %d\n"), rc); From 1d54fa9f62104999beb86ed176eabc2a6f44d452 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Wed, 14 Jun 2017 11:30:25 +0700 Subject: [PATCH 03/26] Fix HTTP server response to handle 404 Not Found * tests/libtest.c: Fix HTTP server response to handle 404 Not Found status code with empty body --- tests/libtest.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/libtest.c b/tests/libtest.c index b0a0f08c6..0ddb3e860 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -291,6 +291,12 @@ static int _answer_to_connection(void *cls, } } + // 404 with empty "body" + if (found == 0) { + response = MHD_create_response_from_buffer(0, (void *) "", MHD_RESPMEM_PERSISTENT); + ret = MHD_queue_response(connection, MHD_HTTP_NOT_FOUND, response); + } + MHD_destroy_response(response); return ret; } From cdd6aaf8dc637927cd7f9dfbf4a44975979d6223 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Fri, 16 Jun 2017 11:17:22 +0700 Subject: [PATCH 04/26] Fix requested URL which contains query string * tests/libtest.c: Fix requested URL which contains query string --- tests/libtest.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/libtest.c b/tests/libtest.c index 0ddb3e860..0f74e7c78 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -92,6 +92,14 @@ struct MHD_Daemon *httpdaemon; #endif +// for passing URL query string +struct query_string { + wget_buffer_t + *params; + int + it; +}; + static void sigterm_handler(int sig G_GNUC_WGET_UNUSED) { terminate = 1; @@ -267,6 +275,33 @@ static void *_http_server_thread(void *ctx) } #ifdef WITH_MICROHTTPD +static int _print_query_string(void *cls, enum MHD_ValueKind kind, + const char *key, + const char *value) +{ + struct query_string *query = cls; + + if (key && query->it == 0) { + wget_buffer_strcpy(query->params, "?"); + wget_buffer_strcat(query->params, key); + if (value) { + wget_buffer_strcat(query->params, "="); + wget_buffer_strcat(query->params, value); + } + } + if (key && query->it != 0) { + wget_buffer_strcat(query->params, "&"); + wget_buffer_strcat(query->params, key); + if (value) { + wget_buffer_strcat(query->params, "="); + wget_buffer_strcat(query->params, value); + } + } + + query->it++; + return MHD_YES; +} + static int _answer_to_connection(void *cls, struct MHD_Connection *connection, const char *url, @@ -275,12 +310,25 @@ static int _answer_to_connection(void *cls, const char *upload_data, size_t *upload_data_size, void **con_cls) { struct MHD_Response *response; + struct query_string query; int ret; + // get query string + query.params = wget_buffer_alloc(1024); + query.it = 0; + MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &_print_query_string, &query); + + // append query string into URL + wget_buffer_t *url_full = wget_buffer_alloc(1024); + wget_buffer_strcpy(url_full, url); + if (query.params->data) + wget_buffer_strcat(url_full, query.params->data); + wget_buffer_free(&query.params); + // it1 = iteration for urls data unsigned int it1, found = 0; for (it1 = 0; it1 < nurls; it1++) { - if (!strcmp(url, urls[it1].name)) + if (!strcmp(url_full->data, urls[it1].name)) { response = MHD_create_response_from_buffer(strlen(urls[it1].body), (void *) urls[it1].body, MHD_RESPMEM_MUST_COPY); @@ -297,6 +345,7 @@ static int _answer_to_connection(void *cls, ret = MHD_queue_response(connection, MHD_HTTP_NOT_FOUND, response); } + wget_buffer_free(&url_full); MHD_destroy_response(response); return ret; } From 9b4943a09154e073f5514c8124076d53be3a56fc Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sat, 17 Jun 2017 10:55:38 +0700 Subject: [PATCH 05/26] Fix HTTP server to get available port * tests/libtest.c: Automatically get available port for HTTP server using Libmicrohttpd function --- tests/libtest.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/libtest.c b/tests/libtest.c index 0f74e7c78..283876ea9 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -49,6 +49,11 @@ #include #endif +#include +#include +#include +#include + static wget_thread_t https_server_tid, ftp_server_tid, @@ -366,6 +371,47 @@ static int _http_server_start(void) if (!httpdaemon) return 1; + // get open random port number + if (0) {} +#if MHD_VERSION >= 0x00095501 + else if (MHD_is_feature_supported(MHD_FEATURE_AUTODETECT_BIND_PORT != MHD_NO)) + { + const union MHD_DaemonInfo *dinfo; + dinfo = MHD_get_daemon_info(httpdaemon, MHD_DAEMON_INFO_BIND_PORT); + if (dinfo == NULL || dinfo->port == 0) + { + return 1; + } + port_num = (int)dinfo->port; + http_server_port = port_num; + } +#endif /* MHD_VERSION >= 0x00095501 */ + else + { + const union MHD_DaemonInfo *dinfo; + MHD_socket sock_fd; + dinfo = MHD_get_daemon_info(httpdaemon, MHD_DAEMON_INFO_LISTEN_FD); + + if (dinfo == NULL) + { + return 1; + } + sock_fd = dinfo->listen_fd; + + struct sockaddr_storage addr_store; + struct sockaddr *addr = (struct sockaddr *)&addr_store; + socklen_t addr_len = sizeof(addr_store); + char s_port[NI_MAXSERV]; + + // get automatic retrieved port number + if (getsockname(sock_fd, addr, &addr_len) == 0) { + if (getnameinfo(addr, addr_len, NULL, 0, s_port, sizeof(s_port), NI_NUMERICSERV) == 0) + port_num = atoi(s_port); + http_server_port = port_num; + } + + } + return 0; } #endif From 820fce6c66f737edf5cb92faff4a1c68f8759289 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sun, 18 Jun 2017 11:36:55 +0700 Subject: [PATCH 06/26] Fix HTTP server response to include available header * tests/libtest.c: Fix HTTP server response to include available header. This solved many cases which involving proper HTTP header response such as for Chunked "Transfer-Encoding" and "Redirection" --- tests/libtest.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/libtest.c b/tests/libtest.c index 283876ea9..5f85d3a6a 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -54,6 +54,7 @@ #include #include + static wget_thread_t https_server_tid, ftp_server_tid, @@ -339,6 +340,20 @@ static int _answer_to_connection(void *cls, (void *) urls[it1].body, MHD_RESPMEM_MUST_COPY); ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + // add available headers + if (*urls[it1].headers) { + // it2 = iteration for headers + for (unsigned int it2 = 0; urls[it1].headers[it2] != NULL; it2++) { + const char *header = urls[it1].headers[it2]; + if (header) { + const char *header_value = strchr(header, ':'); + const char *header_key = wget_strmemdup(header, header_value - header); + MHD_add_response_header(response, header_key, header_value + 2); + wget_xfree(header_key); + } + } + } + it1 = nurls; found = 1; } From 0f89a7e9e4f5798886c1e4dcd561efb253ceb636 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sun, 18 Jun 2017 11:36:55 +0700 Subject: [PATCH 07/26] Fix HTTP server response to handle directory creation * tests/libtest.c: Fix HTTP server response regarding directory creation on recursive download --- tests/libtest.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/libtest.c b/tests/libtest.c index 5f85d3a6a..3f2c9c25f 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -54,7 +54,6 @@ #include #include - static wget_thread_t https_server_tid, ftp_server_tid, @@ -281,6 +280,16 @@ static void *_http_server_thread(void *ctx) } #ifdef WITH_MICROHTTPD +static char *_scan_directory(const char* data) +{ + char *path = strchr(data, '/'); + if (path != 0) { + return path; + } + else + return NULL; +} + static int _print_query_string(void *cls, enum MHD_ValueKind kind, const char *key, const char *value) @@ -334,6 +343,11 @@ static int _answer_to_connection(void *cls, // it1 = iteration for urls data unsigned int it1, found = 0; for (it1 = 0; it1 < nurls; it1++) { + // create default page for directory without index page + char *dir = _scan_directory(url_full->data + 1); + if (dir != 0 && !strcmp(dir, "/")) + wget_buffer_strcat(url_full, "index.html"); + if (!strcmp(url_full->data, urls[it1].name)) { response = MHD_create_response_from_buffer(strlen(urls[it1].body), From 7a5304ad60ab087e458ddc23b8a3ca601be188ae Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Wed, 21 Jun 2017 11:36:55 +0700 Subject: [PATCH 08/26] Fix remote URL to handle IRI object * tests/libtest.c: Fix remote URL which contains IRI object using Libmicrohttpd function (MHD_http_unescape) --- tests/libtest.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/libtest.c b/tests/libtest.c index 3f2c9c25f..9e85bc0b8 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -348,7 +348,12 @@ static int _answer_to_connection(void *cls, if (dir != 0 && !strcmp(dir, "/")) wget_buffer_strcat(url_full, "index.html"); - if (!strcmp(url_full->data, urls[it1].name)) + // convert remote url into escaped char for iri encoding + wget_buffer_t *url_iri = wget_buffer_alloc(1024); + wget_buffer_strcpy(url_iri, urls[it1].name); + MHD_http_unescape(url_iri->data); + + if (!strcmp(url_full->data, url_iri->data)) { response = MHD_create_response_from_buffer(strlen(urls[it1].body), (void *) urls[it1].body, MHD_RESPMEM_MUST_COPY); @@ -371,6 +376,8 @@ static int _answer_to_connection(void *cls, it1 = nurls; found = 1; } + + wget_buffer_free(&url_iri); } // 404 with empty "body" @@ -818,6 +825,7 @@ void wget_test_start_server(int first_key, ...) wget_ssl_set_config_string(WGET_SSL_CERT_FILE, SRCDIR "/certs/x509-server-cert.pem"); wget_ssl_set_config_string(WGET_SSL_KEY_FILE, SRCDIR "/certs/x509-server-key.pem"); + // init HTTPS server socket https_parent_tcp = wget_tcp_init(); wget_tcp_set_ssl(https_parent_tcp, 1); // switch SSL on From 78c2b242a24b59e348043eedb045bdb9465925cc Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Thu, 22 Jun 2017 11:36:55 +0700 Subject: [PATCH 09/26] Fix empty requested URL * tests/libtest.c: Fix empty requested URL by redirect default page to index.html --- tests/libtest.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/libtest.c b/tests/libtest.c index 9e85bc0b8..0adb0ecae 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -340,6 +340,10 @@ static int _answer_to_connection(void *cls, wget_buffer_strcat(url_full, query.params->data); wget_buffer_free(&query.params); + // default page to index.html + if (!strcmp(url_full->data, "/")) + wget_buffer_strcat(url_full, "index.html"); + // it1 = iteration for urls data unsigned int it1, found = 0; for (it1 = 0; it1 < nurls; it1++) { @@ -825,7 +829,6 @@ void wget_test_start_server(int first_key, ...) wget_ssl_set_config_string(WGET_SSL_CERT_FILE, SRCDIR "/certs/x509-server-cert.pem"); wget_ssl_set_config_string(WGET_SSL_KEY_FILE, SRCDIR "/certs/x509-server-key.pem"); - // init HTTPS server socket https_parent_tcp = wget_tcp_init(); wget_tcp_set_ssl(https_parent_tcp, 1); // switch SSL on From 3a2c02c70ec3871ec668b35ca7cc883275cd69b6 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Fri, 23 Jun 2017 11:36:55 +0700 Subject: [PATCH 10/26] Fix empty requested URL regarding IDN hostname * tests/libtest.c: Fix empty requested URL by redirect default page to index.html regarding IDN hostname --- tests/libtest.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/libtest.c b/tests/libtest.c index 0adb0ecae..16902d430 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -290,6 +290,15 @@ static char *_scan_directory(const char* data) return NULL; } +static char *_parse_hostname(const char* data) +{ + if (!wget_strncasecmp_ascii(data, "http://", 7)) { + char *path = strchr(data += 7, '/'); + return path; + } else + return NULL; +} + static int _print_query_string(void *cls, enum MHD_ValueKind kind, const char *key, const char *value) @@ -352,6 +361,11 @@ static int _answer_to_connection(void *cls, if (dir != 0 && !strcmp(dir, "/")) wget_buffer_strcat(url_full, "index.html"); + // create default page for hostname without index page + char *host = _parse_hostname(url_full->data); + if (host != 0 && !strcmp(host, "/")) + wget_buffer_strcat(url_full, "index.html"); + // convert remote url into escaped char for iri encoding wget_buffer_t *url_iri = wget_buffer_alloc(1024); wget_buffer_strcpy(url_iri, urls[it1].name); From 6dd43d7a226f1454694131a7b511fa17c0baea04 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sun, 25 Jun 2017 11:36:55 +0700 Subject: [PATCH 11/26] Fix requested URL query string which contains space * tests/libtest.c: Fix requested URL query string which contains space --- tests/libtest.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/libtest.c b/tests/libtest.c index 16902d430..b1ee5f714 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -299,7 +299,14 @@ static char *_parse_hostname(const char* data) return NULL; } -static int _print_query_string(void *cls, enum MHD_ValueKind kind, +static void _replace_space_with_plus(wget_buffer_t *buf, const char *data) +{ + for (; *data; data++) + wget_buffer_memcat(buf, *data == ' ' ? "+" : data, 1); +} + +static int _print_query_string(void *cls, + enum MHD_ValueKind kind, const char *key, const char *value) { @@ -307,18 +314,18 @@ static int _print_query_string(void *cls, enum MHD_ValueKind kind, if (key && query->it == 0) { wget_buffer_strcpy(query->params, "?"); - wget_buffer_strcat(query->params, key); + _replace_space_with_plus(query->params, key); if (value) { wget_buffer_strcat(query->params, "="); - wget_buffer_strcat(query->params, value); + _replace_space_with_plus(query->params, value); } } if (key && query->it != 0) { wget_buffer_strcat(query->params, "&"); - wget_buffer_strcat(query->params, key); + _replace_space_with_plus(query->params, key); if (value) { wget_buffer_strcat(query->params, "="); - wget_buffer_strcat(query->params, value); + _replace_space_with_plus(query->params, value); } } From d735cf96e212f172d9073c73622f264e92da345d Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Wed, 5 Jul 2017 08:56:46 +0700 Subject: [PATCH 12/26] Fix HTTP server response to handle If-Modified-Since * tests/libtest.c: Fix HTTP server response to handle If-Modified-Since --- tests/libtest.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/libtest.c b/tests/libtest.c index b1ee5f714..84d44ffc3 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -343,12 +343,21 @@ static int _answer_to_connection(void *cls, struct MHD_Response *response; struct query_string query; int ret; + time_t modified; + const char *modified_val; // get query string query.params = wget_buffer_alloc(1024); query.it = 0; MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &_print_query_string, &query); + // get if-modified-since header + modified_val = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, + MHD_HTTP_HEADER_IF_MODIFIED_SINCE); + modified = 0; + if (modified_val) + modified = wget_http_parse_full_date(modified_val); + // append query string into URL wget_buffer_t *url_full = wget_buffer_alloc(1024); wget_buffer_strcpy(url_full, url); @@ -380,9 +389,14 @@ static int _answer_to_connection(void *cls, if (!strcmp(url_full->data, url_iri->data)) { + if (modified && urls[it1].modified <= modified) { + response = MHD_create_response_from_buffer(0, (void *) "", MHD_RESPMEM_PERSISTENT); + ret = MHD_queue_response(connection, MHD_HTTP_NOT_MODIFIED, response); + } else { response = MHD_create_response_from_buffer(strlen(urls[it1].body), (void *) urls[it1].body, MHD_RESPMEM_MUST_COPY); ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + } // add available headers if (*urls[it1].headers) { From 2a60315976ea0874ce147ebcbec7ea4fcacb48bf Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sun, 9 Jul 2017 22:58:03 +0700 Subject: [PATCH 13/26] Fix HTTP server response to handle Byte serving * tests/libtest.c: Fix HTTP server response to handle Byte serving --- tests/libtest.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/libtest.c b/tests/libtest.c index 84d44ffc3..c931cdfd6 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -333,6 +333,22 @@ static int _print_query_string(void *cls, return MHD_YES; } +static int _print_header_range(void *cls, enum MHD_ValueKind kind, + const char *key, + const char *value) +{ + wget_buffer_t *header_range = cls; + + if (!strcmp(key, MHD_HTTP_HEADER_RANGE)) { + wget_buffer_strcpy(header_range, key); + if (value) { + wget_buffer_strcat(header_range, value); + } + } + + return MHD_YES; +} + static int _answer_to_connection(void *cls, struct MHD_Connection *connection, const char *url, @@ -344,7 +360,10 @@ static int _answer_to_connection(void *cls, struct query_string query; int ret; time_t modified; - const char *modified_val; + const char *modified_val, *to_bytes_string; + ssize_t from_bytes, to_bytes; + size_t body_len; + char content_len[100], content_range[100]; // get query string query.params = wget_buffer_alloc(1024); @@ -358,6 +377,23 @@ static int _answer_to_connection(void *cls, if (modified_val) modified = wget_http_parse_full_date(modified_val); + // get header range + wget_buffer_t *header_range = wget_buffer_alloc(1024); + if (!strcmp(method, "GET")) + MHD_get_connection_values(connection, MHD_HEADER_KIND, &_print_header_range, header_range); + + from_bytes = to_bytes = body_len = 0; + if (*header_range->data) { + const char *from_bytes_string; + const char *range_string = strchr(header_range->data, '='); + to_bytes_string = strchr(range_string, '-'); + if (strcmp(to_bytes_string, "-")) + to_bytes = (ssize_t) atoi(to_bytes_string + 1); + from_bytes_string = wget_strmemdup(range_string, to_bytes_string - range_string); + from_bytes = (ssize_t) atoi(from_bytes_string + 1); + wget_xfree(from_bytes_string); + } + // append query string into URL wget_buffer_t *url_full = wget_buffer_alloc(1024); wget_buffer_strcpy(url_full, url); @@ -392,6 +428,30 @@ static int _answer_to_connection(void *cls, if (modified && urls[it1].modified <= modified) { response = MHD_create_response_from_buffer(0, (void *) "", MHD_RESPMEM_PERSISTENT); ret = MHD_queue_response(connection, MHD_HTTP_NOT_MODIFIED, response); + } + else if (*header_range->data) + { + if (!strcmp(to_bytes_string, "-")) + to_bytes = strlen(urls[it1].body) - 1; + body_len = to_bytes - from_bytes + 1; + + if (from_bytes > to_bytes || from_bytes >= (int) strlen(urls[it1].body)) { + response = MHD_create_response_from_buffer(0, (void *) "", MHD_RESPMEM_PERSISTENT); +#ifdef MHD_HTTP_RANGE_NOT_SATISFIABLE + ret = MHD_queue_response(connection, MHD_HTTP_RANGE_NOT_SATISFIABLE, response); +#else + ret = MHD_queue_response(connection, MHD_HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, response); +#endif + } else { + response = MHD_create_response_from_buffer(body_len, + (void *) (urls[it1].body + from_bytes), MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); + sprintf(content_range, "%zd-%zd/%zu", from_bytes, to_bytes, body_len); + MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_RANGE, content_range); + sprintf(content_len, "%zu", body_len); + MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_LENGTH, content_len); + ret = MHD_queue_response(connection, MHD_HTTP_PARTIAL_CONTENT, response); + } } else { response = MHD_create_response_from_buffer(strlen(urls[it1].body), (void *) urls[it1].body, MHD_RESPMEM_MUST_COPY); @@ -426,6 +486,7 @@ static int _answer_to_connection(void *cls, } wget_buffer_free(&url_full); + wget_buffer_free(&header_range); MHD_destroy_response(response); return ret; } From 0f72086cf9f5f960f0870f4461210145132f2c7c Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Mon, 10 Jul 2017 23:46:30 +0700 Subject: [PATCH 14/26] Fix HTTP server response to handle 404 Not Found * tests/libtest.c: Fix HTTP server response to handle 404 Not Found status code with non-empty body --- tests/libtest.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/libtest.c b/tests/libtest.c index c931cdfd6..692f8b09b 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -425,6 +425,17 @@ static int _answer_to_connection(void *cls, if (!strcmp(url_full->data, url_iri->data)) { + // 404 with non-empty "body" + if (!wget_strcmp(urls[it1].code, "404 Not exist")) + { + response = MHD_create_response_from_buffer(strlen(urls[it1].body), + (void *) urls[it1].body, MHD_RESPMEM_MUST_COPY); + ret = MHD_queue_response(connection, MHD_HTTP_NOT_FOUND, response); + wget_buffer_free(&url_iri); + found = 1; + break; + } + if (modified && urls[it1].modified <= modified) { response = MHD_create_response_from_buffer(0, (void *) "", MHD_RESPMEM_PERSISTENT); ret = MHD_queue_response(connection, MHD_HTTP_NOT_MODIFIED, response); From 5d909a2e0630bdf9474f1a325b3624c96916f138 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Tue, 11 Jul 2017 01:07:11 +0700 Subject: [PATCH 15/26] Fix HTTP server response to handle 302 redirection * tests/libtest.c: Fix HTTP server response to handle 302 status code --- tests/libtest.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/libtest.c b/tests/libtest.c index 692f8b09b..85d97c224 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -54,6 +54,7 @@ #include #include + static wget_thread_t https_server_tid, ftp_server_tid, @@ -436,6 +437,27 @@ static int _answer_to_connection(void *cls, break; } + // redirection + if (!wget_strcmp(urls[it1].code, "302 Redirect") || + !wget_strcmp(urls[it1].code, "302 Not found")) + { + response = MHD_create_response_from_buffer(0, (void *) "", MHD_RESPMEM_PERSISTENT); + // it2 = iteration for headers + for (unsigned int it2 = 0; urls[it1].headers[it2] != NULL; it2++) { + const char *header = urls[it1].headers[it2]; + if (header) { + const char *header_value = strchr(header, ':'); + const char *header_key = wget_strmemdup(header, header_value - header); + MHD_add_response_header(response, header_key, header_value + 2); + wget_xfree(header_key); + } + } + ret = MHD_queue_response(connection, MHD_HTTP_FOUND, response); + wget_buffer_free(&url_iri); + found = 1; + break; + } + if (modified && urls[it1].modified <= modified) { response = MHD_create_response_from_buffer(0, (void *) "", MHD_RESPMEM_PERSISTENT); ret = MHD_queue_response(connection, MHD_HTTP_NOT_MODIFIED, response); From efcc1301ee70b8e79555eaa5b84b19d431e3a412 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Thu, 13 Jul 2017 04:28:44 +0700 Subject: [PATCH 16/26] Fix HTTP server to handle basic authentication * tests/libtest.c: Add basic authentication test handler * tests/test-auth-basic.c: Add test for wrong credentials --- tests/libtest.c | 21 ++++++++- tests/test-auth-basic.c | 102 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/tests/libtest.c b/tests/libtest.c index 85d97c224..a4fb345f2 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -54,7 +54,6 @@ #include #include - static wget_thread_t https_server_tid, ftp_server_tid, @@ -458,6 +457,26 @@ static int _answer_to_connection(void *cls, break; } + // basic authentication + if (!wget_strcmp(urls[it1].auth_method, "Basic")) { + char *pass = NULL; + char *user = MHD_basic_auth_get_username_password(connection, &pass); + if ((user == NULL && pass == NULL) || + (wget_strcmp(user, urls[it1].auth_username) || + wget_strcmp(pass, urls[it1].auth_password))) { + response = MHD_create_response_from_buffer(strlen ("DENIED"), + (void *) "DENIED", MHD_RESPMEM_PERSISTENT); + ret = MHD_queue_basic_auth_fail_response(connection, "basic@example.com", response); + free(user); + free(pass); + wget_buffer_free(&url_iri); + found = 1; + break; + } + free(user); + free(pass); + } + if (modified && urls[it1].modified <= modified) { response = MHD_create_response_from_buffer(0, (void *) "", MHD_RESPMEM_PERSISTENT); ret = MHD_queue_response(connection, MHD_HTTP_NOT_MODIFIED, response); diff --git a/tests/test-auth-basic.c b/tests/test-auth-basic.c index 9e760b84c..815d1ce4f 100644 --- a/tests/test-auth-basic.c +++ b/tests/test-auth-basic.c @@ -84,5 +84,107 @@ int main(void) { NULL } }, 0); + // wrong credentials + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --password=" password, + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=\"\" --password=" password, + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=\"whatever\" --password=" password, + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=" username, + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=" username " --password=\"\"", + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=" username " --password=\"whatever\"", + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d", + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=\"\" --password=\"\"", + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=\"whatever\" --password=\"whatever\"", + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + + // use --auth-no-challenge options + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=" username " --password=" password " --auth-no-challenge", + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { urls[0].name + 1, urls[0].body }, + { NULL } }, + 0); + + wget_test( +// WGET_TEST_KEEP_TMPFILES, 1, + WGET_TEST_OPTIONS, "-d --user=" username " --password=\"whatever\" --auth-no-challenge", + WGET_TEST_REQUEST_URL, urls[0].name + 1, + WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { + { NULL } }, + 0); + exit(0); } From a00e61cabb3ffaa7e63b35263fde691e585794cc Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Fri, 14 Jul 2017 04:28:44 +0700 Subject: [PATCH 17/26] * tests/libtest.c: Add HTTPS server handler --- tests/libtest.c | 92 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 15 deletions(-) diff --git a/tests/libtest.c b/tests/libtest.c index a4fb345f2..db8cb9e79 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -94,7 +94,8 @@ static char #ifdef WITH_MICROHTTPD // MHD_Daemon instance struct MHD_Daemon - *httpdaemon; + *httpdaemon, + *httpsdaemon; #endif // for passing URL query string @@ -105,6 +106,15 @@ struct query_string { it; }; +char + *key_pem, + *cert_pem; + +enum SERVER_MODE { + HTTP_MODE, + HTTPS_MODE +}; + static void sigterm_handler(int sig G_GNUC_WGET_UNUSED) { terminate = 1; @@ -546,40 +556,82 @@ static int _answer_to_connection(void *cls, static void _http_server_stop(void) { MHD_stop_daemon(httpdaemon); + MHD_stop_daemon(httpsdaemon); + + free(key_pem); + free(cert_pem); } -static int _http_server_start(void) +static int _http_server_start(int SERVER_MODE) { int port_num = 0; - httpdaemon = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, - port_num, NULL, NULL, &_answer_to_connection, NULL, NULL, - MHD_OPTION_END); + if (SERVER_MODE == HTTP_MODE) { + httpdaemon = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, + port_num, NULL, NULL, &_answer_to_connection, NULL, NULL, + MHD_OPTION_END); + + if (!httpdaemon) + return 1; + } else if (SERVER_MODE == HTTPS_MODE) { + size_t size; + + key_pem = wget_read_file(SRCDIR "/certs/x509-server-key.pem", &size); + cert_pem = wget_read_file(SRCDIR "/certs/x509-server-cert.pem", &size); + + if ((key_pem == NULL) || (cert_pem == NULL)) + { + printf("The key/certificate files could not be read.\n"); + + return 1; + } + +#ifdef MHD_USE_TLS + httpsdaemon = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY | MHD_USE_TLS, +#else + httpsdaemon = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY | MHD_USE_SSL, +#endif + port_num, NULL, NULL, &_answer_to_connection, NULL, + MHD_OPTION_HTTPS_MEM_KEY, key_pem, + MHD_OPTION_HTTPS_MEM_CERT, cert_pem, + MHD_OPTION_END); + + if (!httpsdaemon) { + printf("Cannot start the HTTPS server.\n"); - if (!httpdaemon) - return 1; + return 1; + } + } // get open random port number if (0) {} #if MHD_VERSION >= 0x00095501 - else if (MHD_is_feature_supported(MHD_FEATURE_AUTODETECT_BIND_PORT != MHD_NO)) + else if (MHD_NO != MHD_is_feature_supported(MHD_FEATURE_AUTODETECT_BIND_PORT)) { const union MHD_DaemonInfo *dinfo; - dinfo = MHD_get_daemon_info(httpdaemon, MHD_DAEMON_INFO_BIND_PORT); + if (SERVER_MODE == HTTP_MODE) + dinfo = MHD_get_daemon_info(httpdaemon, MHD_DAEMON_INFO_BIND_PORT); + else if (SERVER_MODE == HTTPS_MODE) + dinfo = MHD_get_daemon_info(httpsdaemon, MHD_DAEMON_INFO_BIND_PORT); if (dinfo == NULL || dinfo->port == 0) { return 1; } port_num = (int)dinfo->port; - http_server_port = port_num; + if (SERVER_MODE == HTTP_MODE) + http_server_port = port_num; + else if (SERVER_MODE == HTTPS_MODE) + https_server_port = port_num; } #endif /* MHD_VERSION >= 0x00095501 */ else { const union MHD_DaemonInfo *dinfo; MHD_socket sock_fd; - dinfo = MHD_get_daemon_info(httpdaemon, MHD_DAEMON_INFO_LISTEN_FD); - + if (SERVER_MODE == HTTP_MODE) + dinfo = MHD_get_daemon_info(httpdaemon, MHD_DAEMON_INFO_LISTEN_FD); + else if (SERVER_MODE == HTTPS_MODE) + dinfo = MHD_get_daemon_info(httpsdaemon, MHD_DAEMON_INFO_LISTEN_FD); if (dinfo == NULL) { return 1; @@ -593,9 +645,13 @@ static int _http_server_start(void) // get automatic retrieved port number if (getsockname(sock_fd, addr, &addr_len) == 0) { - if (getnameinfo(addr, addr_len, NULL, 0, s_port, sizeof(s_port), NI_NUMERICSERV) == 0) + if (getnameinfo(addr, addr_len, NULL, 0, s_port, sizeof(s_port), NI_NUMERICSERV) == 0) { port_num = atoi(s_port); - http_server_port = port_num; + if (SERVER_MODE == HTTP_MODE) + http_server_port = port_num; + else if (SERVER_MODE == HTTPS_MODE) + https_server_port = port_num; + } } } @@ -1007,8 +1063,14 @@ void wget_test_start_server(int first_key, ...) #ifdef WITH_MICROHTTPD // start HTTP server - if ((rc = _http_server_start()) != 0) + if ((rc = _http_server_start(HTTP_MODE)) != 0) wget_error_printf_exit(_("Failed to start HTTP server, error %d\n"), rc); + +#ifdef WITH_GNUTLS + // start HTTPS server + if ((rc = _http_server_start(HTTPS_MODE)) != 0) + wget_error_printf_exit(_("Failed to start HTTPS server, error %d\n"), rc); +#endif #endif // now replace {{port}} in the body by the actual server port From d289f78b005a41257b5bfeadf693520e0a05c803 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Wed, 19 Jul 2017 05:16:43 +0700 Subject: [PATCH 18/26] * tests/libtest.c: Fix -Wunused-param caused by MHD API --- tests/libtest.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/libtest.c b/tests/libtest.c index db8cb9e79..488207faa 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -316,7 +316,7 @@ static void _replace_space_with_plus(wget_buffer_t *buf, const char *data) } static int _print_query_string(void *cls, - enum MHD_ValueKind kind, + enum MHD_ValueKind kind G_GNUC_WGET_UNUSED, const char *key, const char *value) { @@ -343,7 +343,8 @@ static int _print_query_string(void *cls, return MHD_YES; } -static int _print_header_range(void *cls, enum MHD_ValueKind kind, +static int _print_header_range(void *cls, + enum MHD_ValueKind kind G_GNUC_WGET_UNUSED, const char *key, const char *value) { @@ -359,12 +360,14 @@ static int _print_header_range(void *cls, enum MHD_ValueKind kind, return MHD_YES; } -static int _answer_to_connection(void *cls, +static int _answer_to_connection(void *cls G_GNUC_WGET_UNUSED, struct MHD_Connection *connection, const char *url, const char *method, - const char *version, - const char *upload_data, size_t *upload_data_size, void **con_cls) + const char *version G_GNUC_WGET_UNUSED, + const char *upload_data G_GNUC_WGET_UNUSED, + size_t *upload_data_size G_GNUC_WGET_UNUSED, + void **con_cls G_GNUC_WGET_UNUSED) { struct MHD_Response *response; struct query_string query; From caa1d2827bb9a7cb8bc0bed55b06d88e63377222 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Fri, 4 Aug 2017 15:18:04 +0700 Subject: [PATCH 19/26] Fix HTTP server to handle chunked transfer encoding * tests/libtest.c: Fix HTTP server to handle chunked transfer encoding * tests/test-bad-chunk.c: Modify test to conform with new server * tests/test-chunked.c: Likewise --- tests/libtest.c | 81 +++++++++++++++++++++++++++++++++++++++++- tests/test-bad-chunk.c | 8 ++--- tests/test-chunked.c | 4 +-- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/tests/libtest.c b/tests/libtest.c index 488207faa..d524c4464 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -360,6 +360,41 @@ static int _print_header_range(void *cls, return MHD_YES; } +struct ResponseContentCallbackParam +{ + const char *response_data; + size_t response_size; +}; + +static ssize_t _callback (void *cls, + uint64_t pos, + char *buf, + size_t buf_size) +{ + size_t size_to_copy; + struct ResponseContentCallbackParam *const param = + (struct ResponseContentCallbackParam *)cls; + + if (pos >= param->response_size) + return MHD_CONTENT_READER_END_OF_STREAM; + + // divide data into two chunks + buf_size = (param->response_size / 2) + 1; + if (buf_size < (param->response_size - pos)) + size_to_copy = buf_size; + else + size_to_copy = param->response_size - pos; + + memcpy (buf, param->response_data + pos, size_to_copy); + + return size_to_copy; +} + +static void _free_callback_param(void *cls) +{ + free(cls); +} + static int _answer_to_connection(void *cls G_GNUC_WGET_UNUSED, struct MHD_Connection *connection, const char *url, @@ -419,7 +454,7 @@ static int _answer_to_connection(void *cls G_GNUC_WGET_UNUSED, wget_buffer_strcat(url_full, "index.html"); // it1 = iteration for urls data - unsigned int it1, found = 0; + unsigned int it1, found = 0, chunked = 0; for (it1 = 0; it1 < nurls; it1++) { // create default page for directory without index page char *dir = _scan_directory(url_full->data + 1); @@ -438,6 +473,50 @@ static int _answer_to_connection(void *cls G_GNUC_WGET_UNUSED, if (!strcmp(url_full->data, url_iri->data)) { + // chunked encoding + if (!wget_strcmp(urls[it1].name + 3, "bad.txt")) + { + response = MHD_create_response_from_buffer(strlen(urls[it1].body), + (void *) urls[it1].body, MHD_RESPMEM_MUST_COPY); + ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + MHD_add_response_header(response, "Transfer-Encoding", "chunked"); + MHD_add_response_header(response, "Connection", "close"); + wget_buffer_free(&url_iri); + found = 1; + break; + } + for (int it2 = 0; urls[it1].headers[it2] != NULL; it2++) { + const char *header = urls[it1].headers[it2]; + if (header) { + const char *header_value = strchr(header, ':'); + const char *header_key = wget_strmemdup(header, header_value - header); + if (!strcmp(header_key, "Transfer-Encoding") && + !strcmp(header_value + 2, "chunked")) + chunked = 1; + wget_xfree(header_key); + } + } + if (chunked == 1) { + struct ResponseContentCallbackParam *callback_param; + callback_param = malloc(sizeof(struct ResponseContentCallbackParam)); + + static char response_text[44]; + strcpy(response_text, urls[it1].body); + + callback_param->response_data = response_text; + callback_param->response_size = (sizeof(response_text)/sizeof(char)) - 1; + + response = MHD_create_response_from_callback(MHD_SIZE_UNKNOWN, + 1024, + &_callback, + callback_param, + &_free_callback_param); + ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + wget_buffer_free(&url_iri); + found = 1; + break; + } + // 404 with non-empty "body" if (!wget_strcmp(urls[it1].code, "404 Not exist")) { diff --git a/tests/test-bad-chunk.c b/tests/test-bad-chunk.c index 89bfd765d..7c10a5c68 100644 --- a/tests/test-bad-chunk.c +++ b/tests/test-bad-chunk.c @@ -29,7 +29,7 @@ int main(void) { wget_test_url_t urls[]={ - { .name = "/1.txt", + { .name = "/1.bad.txt", .code = "200 Dontcare", .body = "FFFFFFF4\nGarbage", .headers = { @@ -37,7 +37,7 @@ int main(void) "Transfer-Encoding: chunked", } }, - { .name = "/2.txt", + { .name = "/2.bad.txt", .code = "200 Dontcare", .body = "FFFFFFFE\r\nGarbage", .headers = { @@ -45,7 +45,7 @@ int main(void) "Transfer-Encoding: chunked", } }, - { .name = "/3.txt", + { .name = "/3.bad.txt", .code = "200 Dontcare", .body = "FFFFFFFFFFFFFFF4\r\nGarbage", .headers = { @@ -63,7 +63,7 @@ int main(void) // test negative chunk size (32bit system only) wget_test( WGET_TEST_OPTIONS, "", - WGET_TEST_REQUEST_URLS, "1.txt", "2.txt", "3.txt", NULL, + WGET_TEST_REQUEST_URLS, "1.bad.txt", "2.bad.txt", "3.bad.txt", NULL, WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { urls[0].name + 1, NULL }, // do not check content diff --git a/tests/test-chunked.c b/tests/test-chunked.c index d9a85e514..82e027b81 100644 --- a/tests/test-chunked.c +++ b/tests/test-chunked.c @@ -31,7 +31,7 @@ int main(void) wget_test_url_t urls[]={ { .name = "/1.txt", .code = "200 Dontcare", - .body = "3\r\nabc\r\n2\r\nde\r\n", + .body = "the quick brown fox\njumps over the lazy dog", .headers = { "Content-Type: text/plain", "Transfer-Encoding: chunked", @@ -50,7 +50,7 @@ int main(void) WGET_TEST_REQUEST_URL, "1.txt", WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { - { urls[0].name + 1, "abcde" }, + { urls[0].name + 1, "the quick brown fox\njumps over the lazy dog" }, { NULL } }, 0); From 2f77ea19072ab82e77ad93ba6957b2d1ff65b727 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Wed, 23 Aug 2017 19:16:27 +0700 Subject: [PATCH 20/26] * tests/libtest.c: Add server info into HTTP header --- tests/libtest.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/libtest.c b/tests/libtest.c index d524c4464..9f27dfc08 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -631,6 +631,9 @@ static int _answer_to_connection(void *cls G_GNUC_WGET_UNUSED, wget_buffer_free(&url_full); wget_buffer_free(&header_range); + char server_version[50]; + sprintf(server_version, "Libmicrohttpd/%08x", (unsigned int) MHD_VERSION); + MHD_add_response_header(response, "Server", server_version); MHD_destroy_response(response); return ret; } From 650f65030a773585accd73943ee1273058d9af5f Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sun, 3 Sep 2017 16:52:51 +0700 Subject: [PATCH 21/26] * test/libtest.c: Ensure dinfo variable to be initialized before usage --- tests/libtest.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/libtest.c b/tests/libtest.c index 9f27dfc08..db48d47fb 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -693,12 +693,12 @@ static int _http_server_start(int SERVER_MODE) #if MHD_VERSION >= 0x00095501 else if (MHD_NO != MHD_is_feature_supported(MHD_FEATURE_AUTODETECT_BIND_PORT)) { - const union MHD_DaemonInfo *dinfo; + const union MHD_DaemonInfo *dinfo = NULL; if (SERVER_MODE == HTTP_MODE) dinfo = MHD_get_daemon_info(httpdaemon, MHD_DAEMON_INFO_BIND_PORT); else if (SERVER_MODE == HTTPS_MODE) dinfo = MHD_get_daemon_info(httpsdaemon, MHD_DAEMON_INFO_BIND_PORT); - if (dinfo == NULL || dinfo->port == 0) + if (!dinfo || dinfo->port == 0) { return 1; } @@ -711,16 +711,14 @@ static int _http_server_start(int SERVER_MODE) #endif /* MHD_VERSION >= 0x00095501 */ else { - const union MHD_DaemonInfo *dinfo; + const union MHD_DaemonInfo *dinfo = NULL; MHD_socket sock_fd; if (SERVER_MODE == HTTP_MODE) dinfo = MHD_get_daemon_info(httpdaemon, MHD_DAEMON_INFO_LISTEN_FD); else if (SERVER_MODE == HTTPS_MODE) dinfo = MHD_get_daemon_info(httpsdaemon, MHD_DAEMON_INFO_LISTEN_FD); - if (dinfo == NULL) - { + if (!dinfo) return 1; - } sock_fd = dinfo->listen_fd; struct sockaddr_storage addr_store; From 16367671ec1d32f5926e0958d9ea70794212e15a Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sun, 3 Sep 2017 16:59:17 +0700 Subject: [PATCH 22/26] * test/libtest.c: Remove unnecessary old http server code --- tests/libtest.c | 186 ------------------------------------------------ 1 file changed, 186 deletions(-) diff --git a/tests/libtest.c b/tests/libtest.c index db48d47fb..04523df98 100644 --- a/tests/libtest.c +++ b/tests/libtest.c @@ -38,7 +38,6 @@ #include #include #include -#include #include #include @@ -55,7 +54,6 @@ #include static wget_thread_t - https_server_tid, ftp_server_tid, ftps_server_tid; static int @@ -120,175 +118,6 @@ static void sigterm_handler(int sig G_GNUC_WGET_UNUSED) terminate = 1; } -static void *_http_server_thread(void *ctx) -{ - wget_tcp_t *tcp=NULL, *parent_tcp = ctx; - wget_test_url_t *url = NULL; - char buf[4096], method[32], request_url[256], tag[64], value[256], *p; - ssize_t from_bytes, to_bytes, n; - size_t nbytes, body_len, request_url_length; - unsigned it; - int byterange, authorized; - time_t modified; - -#ifdef _WIN32 - signal(SIGTERM, sigterm_handler); -#else - sigaction(SIGTERM, &(struct sigaction) { .sa_handler = sigterm_handler }, NULL); -#endif - - while (!terminate) { - wget_tcp_deinit(&tcp); - - wget_info_printf("[SERVER] accept...\n"); - if ((tcp = wget_tcp_accept(parent_tcp))) { - wget_info_printf("[SERVER] accepted\n"); - authorized = 0; - - nbytes = 0; - while ((n = wget_tcp_read(tcp, buf + nbytes, sizeof(buf) - 1 - nbytes)) > 0) { - nbytes += n; - buf[nbytes]=0; - wget_info_printf(_("[SERVER] got %zd bytes (total %zu)\n"), n, nbytes); - if (strstr(buf,"\r\n\r\n")) - break; - } - wget_info_printf(_("[SERVER] total %zd bytes (total %zu) (errno=%d)\n"), n, nbytes, errno); - - if (nbytes > 0) { - if (sscanf(buf, "%31s %255s", method, request_url) !=2) { - wget_tcp_printf(tcp, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n"); - continue; - } - - byterange = from_bytes = to_bytes = 0; - modified = 0; - - for (p = strstr(buf, "\r\n"); p && sscanf(p, "\r\n%63[^:]: %255[^\r]", tag, value) == 2; p = strstr(p + 2, "\r\n")) { - if (!wget_strcasecmp_ascii(tag, "Range")) { - if ((byterange = sscanf(value, "bytes=%zd-%zd", &from_bytes, &to_bytes)) < 1) - byterange = 0; - } - else if (url && !wget_strcasecmp_ascii(tag, "Authorization")) { - const char *auth_scheme, *s; - - s=wget_http_parse_token(value, &auth_scheme); - while (c_isblank(*s)) s++; - - if (!wget_strcasecmp_ascii(auth_scheme, "basic")) { - const char *encoded = wget_base64_encode_printf_alloc("%s:%s", url->auth_username, url->auth_password); - - wget_error_printf("Auth check '%s' <-> '%s'\n", encoded, s); - if (!strcmp(encoded, s)) - authorized = 1; - - wget_xfree(encoded); - } - - wget_xfree(auth_scheme); - } - else if (!wget_strcasecmp_ascii(tag, "If-Modified-Since")) { - modified = wget_http_parse_full_date(value); - wget_info_printf("modified = %ld\n", modified); - } - } - - url = NULL; - request_url_length = strlen(request_url); - - if (request_url[request_url_length - 1] == '/') { - // access a directory - for (it = 0; it < nurls; it++) { - if (!strcmp(request_url, urls[it].name) || - (!strncmp(request_url, urls[it].name, request_url_length) && - !strcmp(urls[it].name + request_url_length, "index.html"))) - { - url = &urls[it]; - break; - } - } - } else { - // access a file - for (it = 0; it < nurls; it++) { - // printf("%s %s\n", request_url, urls[it].name); - if (!strcmp(request_url, urls[it].name)) { - url = &urls[it]; - break; - } - } - } - - if (!url) { - wget_tcp_printf(tcp, "HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"); - continue; - } - - if (url->auth_method && !authorized) { - if (!wget_strcasecmp_ascii(url->auth_method, "basic")) - wget_tcp_printf(tcp, - "HTTP/1.1 401 Unauthorized\r\n" \ - "WWW-Authenticate: %s realm=\"Protected Page\"\r\n" \ - "Connection: close\r\n\r\n", - url->auth_method); - else - wget_error_printf(_("Unknown authentication scheme '%s'\n"), url->auth_method); - - continue; - } - - if (modified && url->modified<=modified) { - wget_tcp_printf(tcp,"HTTP/1.1 304 Not Modified\r\n\r\n"); - continue; - } - - if (byterange == 1) { - to_bytes = strlen(url->body) - 1; - } - if (byterange) { - if (from_bytes > to_bytes || from_bytes >= (int)strlen(url->body)) { - wget_tcp_printf(tcp, "HTTP/1.1 416 Range Not Satisfiable\r\nConnection: close\r\n\r\n"); - continue; - } - - // create response - body_len = to_bytes - from_bytes + 1; - nbytes = snprintf(buf, sizeof(buf), "HTTP/1.1 206 Partial Content\r\n"); - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "Content-Length: %zu\r\n", body_len); - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "Accept-Ranges: bytes\r\n"); - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "Content-Range: %zd-%zd/%zu\r\n", from_bytes, to_bytes, body_len); - for (it = 0; it < countof(url->headers) && url->headers[it]; it++) { - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "%s\r\n", url->headers[it]); - } - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "\r\n"); - if (!strcmp(method, "GET") || !strcmp(method, "POST")) - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "%.*s", (int)body_len, url->body + from_bytes); - } else { - // create response - body_len = strlen(url->body ? url->body : ""); - nbytes = snprintf(buf, sizeof(buf), "HTTP/1.1 %s\r\n", url->code ? url->code : "200 OK"); - if (server_send_content_length) - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "Content-Length: %zu\r\n", body_len); - for (it = 0; it < countof(url->headers) && url->headers[it]; it++) { - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "%s\r\n", url->headers[it]); - } - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "\r\n"); - if (!strcmp(method, "GET") || !strcmp(method, "POST")) - nbytes += snprintf(buf + nbytes, sizeof(buf) - nbytes, "%s", url->body ? url->body : ""); - } - - // send response - wget_tcp_write(tcp, buf, nbytes); - } - } else if (!terminate) - wget_error_printf(_("Failed to get connection (%d)\n"), errno); - } - - wget_tcp_deinit(&parent_tcp); - - wget_info_printf("[SERVER] stopped\n"); - return NULL; -} - #ifdef WITH_MICROHTTPD static char *_scan_directory(const char* data) { @@ -952,21 +781,10 @@ void wget_test_stop_server(void) // free resources - needed for valgrind testing terminate = 1; -// pthread_kill(http_server_tid, SIGTERM); -// pthread_kill(https_server_tid, SIGTERM); -// pthread_kill(ftp_server_tid, SIGTERM); -// if (ftps_implicit) -// pthread_kill(ftps_server_tid, SIGTERM); - wget_thread_cancel(https_server_tid); wget_thread_cancel(ftp_server_tid); if (ftps_implicit) wget_thread_cancel(ftps_server_tid); -// wget_thread_join(http_server_tid); -// wget_thread_join(https_server_tid); -// wget_thread_join(ftp_server_tid); -// if (ftps_implicit) -// wget_thread_join(ftps_server_tid); if (chdir("..") != 0) wget_error_printf(_("Failed to chdir ..\n")); @@ -1175,10 +993,6 @@ void wget_test_start_server(int first_key, ...) } } - // start thread for HTTPS - if ((rc = wget_thread_start(&https_server_tid, _http_server_thread, https_parent_tcp, 0)) != 0) - wget_error_printf_exit(_("Failed to start HTTPS server, error %d\n"), rc); - // start thread for FTP if ((rc = wget_thread_start(&ftp_server_tid, _ftp_server_thread, ftp_parent_tcp, 0)) != 0) wget_error_printf_exit(_("Failed to start FTP server, error %d\n"), rc); From 75e1a3ef9500e28abae3a7b3cd0b4c6724e017d2 Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sun, 3 Sep 2017 17:06:21 +0700 Subject: [PATCH 23/26] Fix expected error code regarding wrong authentication test * tests/test-auth-basic.c: Fix expected error code regarding wrong authentication test which use basic scheme --- tests/test-auth-basic.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test-auth-basic.c b/tests/test-auth-basic.c index 815d1ce4f..93f7cb502 100644 --- a/tests/test-auth-basic.c +++ b/tests/test-auth-basic.c @@ -89,7 +89,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --password=" password, WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -98,7 +98,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --user=\"\" --password=" password, WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -107,7 +107,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --user=\"whatever\" --password=" password, WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -116,7 +116,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --user=" username, WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -125,7 +125,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --user=" username " --password=\"\"", WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -134,7 +134,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --user=" username " --password=\"whatever\"", WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -143,7 +143,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d", WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -152,7 +152,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --user=\"\" --password=\"\"", WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -161,7 +161,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --user=\"whatever\" --password=\"whatever\"", WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); @@ -181,7 +181,7 @@ int main(void) // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_OPTIONS, "-d --user=" username " --password=\"whatever\" --auth-no-challenge", WGET_TEST_REQUEST_URL, urls[0].name + 1, - WGET_TEST_EXPECTED_ERROR_CODE, 0, + WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { { NULL } }, 0); From e31d0ad429f0dcfa6fac217138829248f99794da Mon Sep 17 00:00:00 2001 From: Didik Setiawan Date: Sun, 3 Sep 2017 18:20:43 +0700 Subject: [PATCH 24/26] Remove redundant debugging option * tests/test--post-file.c: Remove redundant debugging option * tests/test-auth-basic.c: Likewise * tests/test-metalink.c: Likewise * tests/test-redirection.c: Likewise * tests/test-wget-1.c: Likewise Debugging option already provided in tests/libtest.c, so we don't need to adding it in each test file --- tests/test--post-file.c | 2 +- tests/test-auth-basic.c | 25 ++++++++++++------------- tests/test-metalink.c | 7 ++----- tests/test-redirection.c | 1 - tests/test-wget-1.c | 2 +- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/tests/test--post-file.c b/tests/test--post-file.c index 4225f6106..7ad11ce64 100644 --- a/tests/test--post-file.c +++ b/tests/test--post-file.c @@ -37,7 +37,7 @@ int main(void) // test-i wget_test( - WGET_TEST_OPTIONS, "-d --post-file=nofile", + WGET_TEST_OPTIONS, "--post-file=nofile", WGET_TEST_REQUEST_URL, "", WGET_TEST_EXPECTED_ERROR_CODE, 3, // fails with older Wget (<= 1.14) 0); diff --git a/tests/test-auth-basic.c b/tests/test-auth-basic.c index 93f7cb502..f82a7e8f3 100644 --- a/tests/test-auth-basic.c +++ b/tests/test-auth-basic.c @@ -61,7 +61,7 @@ int main(void) // test-auth-basic wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=" username " --password=" password, + WGET_TEST_OPTIONS, "--user=" username " --password=" password, WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -72,7 +72,7 @@ int main(void) // test-auth-basic with .netrc wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --netrc-file=.netrc", + WGET_TEST_OPTIONS, "--netrc-file=.netrc", WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXISTING_FILES, &(wget_test_file_t []) { @@ -87,7 +87,7 @@ int main(void) // wrong credentials wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --password=" password, + WGET_TEST_OPTIONS, "--password=" password, WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -96,7 +96,7 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=\"\" --password=" password, + WGET_TEST_OPTIONS, "--user=\"\" --password=" password, WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -105,7 +105,7 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=\"whatever\" --password=" password, + WGET_TEST_OPTIONS, "--user=\"whatever\" --password=" password, WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -114,7 +114,7 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=" username, + WGET_TEST_OPTIONS, "--user=" username, WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -123,7 +123,7 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=" username " --password=\"\"", + WGET_TEST_OPTIONS, "--user=" username " --password=\"\"", WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -132,7 +132,7 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=" username " --password=\"whatever\"", + WGET_TEST_OPTIONS, "--user=" username " --password=\"whatever\"", WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -141,7 +141,6 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d", WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -150,7 +149,7 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=\"\" --password=\"\"", + WGET_TEST_OPTIONS, "--user=\"\" --password=\"\"", WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -159,7 +158,7 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=\"whatever\" --password=\"whatever\"", + WGET_TEST_OPTIONS, "--user=\"whatever\" --password=\"whatever\"", WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -169,7 +168,7 @@ int main(void) // use --auth-no-challenge options wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=" username " --password=" password " --auth-no-challenge", + WGET_TEST_OPTIONS, "--user=" username " --password=" password " --auth-no-challenge", WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -179,7 +178,7 @@ int main(void) wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d --user=" username " --password=\"whatever\" --auth-no-challenge", + WGET_TEST_OPTIONS, "--user=" username " --password=\"whatever\" --auth-no-challenge", WGET_TEST_REQUEST_URL, urls[0].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 6, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { diff --git a/tests/test-metalink.c b/tests/test-metalink.c index 3f06084f9..9b4526df0 100644 --- a/tests/test-metalink.c +++ b/tests/test-metalink.c @@ -212,7 +212,6 @@ int main(void) // metalink V3, no pieces wget_test( - // WGET_TEST_OPTIONS, "-d", WGET_TEST_REQUEST_URL, "archive.meta", WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -235,7 +234,6 @@ int main(void) // metalink V4, no pieces wget_test( - // WGET_TEST_OPTIONS, "-d", WGET_TEST_REQUEST_URL, "archive.meta4", WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -245,7 +243,6 @@ int main(void) // metalink V4, two pieces wget_test( - // WGET_TEST_OPTIONS, "-d", WGET_TEST_REQUEST_URL, "archiveP.meta4", WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -273,7 +270,7 @@ int main(void) urls[6].headers[4] = wget_aprintf("Digest: MD5=%s", digest_str); wget_test( -// WGET_TEST_OPTIONS, "-d --tries=1", +// WGET_TEST_OPTIONS, "--tries=1", WGET_TEST_REQUEST_URL, urls[6].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { @@ -289,7 +286,7 @@ int main(void) urls[9].headers[3] = wget_aprintf("Digest: MD5=%s", digest_str); wget_test( - WGET_TEST_OPTIONS, "-d --tries=1", + WGET_TEST_OPTIONS, "--tries=1", WGET_TEST_REQUEST_URL, urls[9].name + 1, WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { diff --git a/tests/test-redirection.c b/tests/test-redirection.c index 206a96540..d907f67c8 100644 --- a/tests/test-redirection.c +++ b/tests/test-redirection.c @@ -55,7 +55,6 @@ int main(void) // test-i wget_test( // WGET_TEST_KEEP_TMPFILES, 1, - WGET_TEST_OPTIONS, "-d", WGET_TEST_REQUEST_URL, "index.html", WGET_TEST_EXPECTED_ERROR_CODE, 0, WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) { diff --git a/tests/test-wget-1.c b/tests/test-wget-1.c index 071b4ca7a..3973348c6 100644 --- a/tests/test-wget-1.c +++ b/tests/test-wget-1.c @@ -555,7 +555,7 @@ int main(void) // test-c-shorter wget_test( - WGET_TEST_OPTIONS, "-d -c", + WGET_TEST_OPTIONS, "-c", WGET_TEST_REQUEST_URL, "dummy.txt", // WGET_TEST_KEEP_TMPFILES, 1, WGET_TEST_EXPECTED_ERROR_CODE, 0, From 81b49e3fa4d5b0c62b934c0bb71bf7a7b140ee41 Mon Sep 17 00:00:00 2001 From: Darshit Shah Date: Tue, 5 Sep 2017 10:30:24 +0200 Subject: [PATCH 25/26] Prevent heap-use-after-free of default_challenges * src/wget_job.h: Add new field to JOB struct, challenges_alloc. This field indicates whether the JOB object owns the challenges struct. * src/job.c(job_free): Free the challenges struct of the job only if it is owned. Else, let the owner handle freeing it. * src/wget.c: Mark the challenges vector as owned when it is taken from the response. If it is the default vector, it is not owned. --- src/job.c | 3 ++- src/wget.c | 5 ++++- src/wget_job.h | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/job.c b/src/job.c index 7844b010c..6e3b44e61 100644 --- a/src/job.c +++ b/src/job.c @@ -42,7 +42,8 @@ void job_free(JOB *job) { - wget_http_free_challenges(&job->challenges); + if (job->challenges_alloc) + wget_http_free_challenges(&job->challenges); wget_http_free_challenges(&job->proxy_challenges); wget_metalink_free(&job->metalink); wget_vector_free(&job->parts); diff --git a/src/wget.c b/src/wget.c index 0b166da4a..7d6d06537 100644 --- a/src/wget.c +++ b/src/wget.c @@ -507,8 +507,10 @@ static void add_url_to_queue(const char *url, wget_iri_t *base, const char *enco if (config.spider || config.chunk_size) new_job->head_first = 1; - if (config.auth_no_challenge) + if (config.auth_no_challenge) { new_job->challenges = config.default_challenges; + new_job->challenges_alloc = false; + } host_add_job(host, new_job); @@ -1315,6 +1317,7 @@ static int process_response_header(wget_http_response_t *resp) } job->challenges = resp->challenges; + job->challenges_alloc = true; resp->challenges = NULL; job->inuse = 0; // try again, but with challenge responses return 1; // stop further processing diff --git a/src/wget_job.h b/src/wget_job.h index d17c3daf0..eb5e8a192 100644 --- a/src/wget_job.h +++ b/src/wget_job.h @@ -85,6 +85,8 @@ struct JOB { auth_failure_count, // number of times server has returned a 401 response mirror_pos, // where to look up the next (metalink) mirror to use piece_pos; // where to look up the next (metalink) piece to download + bool + challenges_alloc; // Indicate whether the challenges vector is owned by the JOB unsigned char inuse : 1, // if job is already in use, 'used_by' holds the thread id of the downloader sitemap : 1, // URL is a sitemap to be scanned in recursive mode From 1ba6eaa2ae41c433097fe4d401d5c4f020171d75 Mon Sep 17 00:00:00 2001 From: Darshit Shah Date: Tue, 5 Sep 2017 10:38:01 +0200 Subject: [PATCH 26/26] * .gitlab-ci.yaml: Set default CFLAGS to retain the human readable symbols --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36733c8db..9140b2334 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ variables: CENTOS7_BUILD: buildenv-centos7 GET_SOURCES_ATTEMPTS: "3" CONFIGURE_BASE_FLAGS: --enable-assert + CFLAGS_DEFAULT: -O0 -g -ggdb3 # We organize the CI runners as thus: @@ -38,6 +39,7 @@ Debian GNU/Linux build: - apt-get update -qq && apt-get install -y -qq libmicrohttpd-dev image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD script: + - export CFLAGS=$CFLAGS_DEFAULT - ./bootstrap && touch .manywarnings - CC=clang ./configure $CONFIGURE_BASE_FLAGS --enable-fsanitize-asan --enable-fsanitize-ubsan -C - make -j$(nproc) check @@ -64,6 +66,7 @@ clang/Fedora: script: - export CC=clang - export CXX=clang++ + - export CFLAGS=$CFLAGS_DEFAULT - ./bootstrap && touch .manywarnings - ./configure $CONFIGURE_BASE_FLAGS --enable-valgrind-tests - make -j$(nproc) check @@ -95,7 +98,8 @@ pages: image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD script: - ./bootstrap - - CFLAGS="-g -O0" ./configure + - export CFLAGS=$CFLAGS_DEFAULT + - ./configure - make -j$(nproc) check-coverage - mkdir -p public # Test suite coverage report