Merge branch 'master' into refactor

.gitlab-ci.yml

@@ -5,6 +5,7 @@ variables:
   CENTOS7_BUILD: buildenv-centos7
   GET_SOURCES_ATTEMPTS: "3"
   CONFIGURE_BASE_FLAGS: --enable-assert
+  CFLAGS_DEFAULT: -O0 -g -ggdb3
 
 
 # We organize the CI runners as thus:
@@ -34,8 +35,11 @@ variables:
 #  * ASan, UBSan, Msan
 #  * distcheck
 Debian GNU/Linux build:
+  before_script:
+    - apt-get update -qq && apt-get install -y -qq libmicrohttpd-dev
   image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
   script:
+    - export CFLAGS=$CFLAGS_DEFAULT
     - ./bootstrap && touch .manywarnings
     - CC=clang ./configure $CONFIGURE_BASE_FLAGS --enable-fsanitize-asan --enable-fsanitize-ubsan -C
     - make -j$(nproc) check
@@ -56,10 +60,13 @@ Debian GNU/Linux build:
       - tests/*.log
 
 clang/Fedora:
+  before_script:
+    - dnf -y install libmicrohttpd-devel
   image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
     - export CC=clang
     - export CXX=clang++
+    - export CFLAGS=$CFLAGS_DEFAULT
     - ./bootstrap && touch .manywarnings
     - ./configure $CONFIGURE_BASE_FLAGS --enable-valgrind-tests
     - make -j$(nproc) check
@@ -91,7 +98,8 @@ pages:
   image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
   script:
     - ./bootstrap
-    - CFLAGS="-g -O0" ./configure
+    - export CFLAGS=$CFLAGS_DEFAULT
+    - ./configure
     - make -j$(nproc) check-coverage
     - mkdir -p public
     # Test suite coverage report
@@ -117,15 +125,28 @@ pages:
 MinGW64:
   image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
   script:
-  - apt-get -y install gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-x86-64-dev pkg-config-mingw-w64-x86-64 win-iconv-mingw-w64-dev wine
-  - ./bootstrap
+  - dpkg --add-architecture i386
+  - apt-get -y update
+  - apt-get -y install gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-x86-64-dev pkg-config-mingw-w64-x86-64 win-iconv-mingw-w64-dev wine wine32 wget
+  - cd ..
   - export PREFIX=x86_64-w64-mingw32
   - export CC=$PREFIX-gcc
-  - export CXX=$PREFIX-g++
   - export CPP=$PREFIX-cpp
   - export RANLIB=$PREFIX-ranlib
   - export PATH="/usr/$PREFIX/bin:$PATH"
   - export CFLAGS="-O2 -Wall -Wno-format"
+  # Install Libmicrohttpd from source
+  - git clone https://gnunet.org/git/libmicrohttpd.git
+  - cd libmicrohttpd
+  - git checkout c0168fde5043523
+  - ./bootstrap
+  - ./configure --build=x86_64-pc-linux-gnu --host=$PREFIX --prefix=/usr/$PREFIX --disable-doc --disable-examples --enable-shared --enable-static
+  - make clean
+  - make -j$(nproc)
+  - make install
+  - cd - && cd wget2
+  - ./bootstrap
+  - export CXX=$PREFIX-g++
   - export WINEPATH="/usr/$PREFIX/bin;/usr/$PREFIX/lib;$PWD/libwget/.libs"
   - ./configure --build=x86_64-pc-linux-gnu --host=$PREFIX --enable-shared --enable-static
   - make clean

.travis.sh

@@ -10,6 +10,15 @@ test "$CC" = "clang" && export CXX="clang++"
 
 if [[ $TRAVIS_OS_NAME = 'osx' ]]; then
 	CONFIGURE_OPTIONS+=("")
+	# Install Libmicrohttpd from source
+	cd ..
+	wget http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.55.tar.gz
+	tar zxf libmicrohttpd-0.9.55.tar.gz && cd libmicrohttpd-0.9.55
+	./configure
+	make clean
+	make -j3
+	make install
+	cd - && cd wget2
 else
 	CONFIGURE_OPTIONS+=("--enable-valgrind-tests")
 fi

.travis_setup.sh

@@ -14,7 +14,13 @@ if [[ "$TRAVIS_OS_NAME" = "osx" ]]; then
 	brew install xz
 	brew install lbzip2
 	brew install lzip
+	brew install libgcrypt
 	brew link --force gettext
 elif [[ "$TRAVIS_OS_NAME" = "linux" ]]; then
+	# Install Libmicrohttpd from source
+	sudo apt-get -y install wget
+	wget http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.55.tar.gz
+	tar zxf libmicrohttpd-0.9.55.tar.gz && cd libmicrohttpd-0.9.55/
+	./configure --prefix=/usr && make -j$(nproc) && sudo make install
 	pip install --user cpp-coveralls
 fi

README.md

@@ -115,6 +115,7 @@ The following packages are needed to build the software
 * flex >= 2.5.35
 * libpsl >= 0.5.0
 * libnghttp2 >= 1.3.0 (optional, if you want HTTP/2 support)
+* libmicrohttpd >= 0.9.51 (optional, if you want to run the test suite)
 
 The versions are recommended, but older versions may also work.
 

configure.ac

@@ -546,6 +546,20 @@ AS_IF([test "x$with_libidn2" != xyes], [
 ])
 AM_CONDITIONAL([WITH_LIBIDN], [test "x$with_libidn" = xyes])
 
+AC_ARG_WITH(libmicrohttpd, AS_HELP_STRING([--without-libmicrohttpd], [disable support for libmicrohttpd]), with_libmicrohttpd=$withval, with_libmicrohttpd=yes)
+AS_IF([test "x$with_libmicrohttpd" != xno], [
+  PKG_CHECK_MODULES(MICROHTTPD, [libmicrohttpd], [
+     with_libmicrohttpd=yes
+     LIBS="$MICROHTTPD_LIBS $LIBS"
+     CFLAGS="$MICROHTTPD_CFLAGS $CFLAGS"
+     AC_DEFINE([WITH_MICROHTTPD], [1], [Use Libmicrohttpd])
+  ], [
+	 with_libmicrohttpd=no
+	 AC_MSG_WARN(*** GNU Libmicrohttpd was not found. You will not be able to run the test suite.)]
+  )
+])
+AM_CONDITIONAL([WITH_MICROHTTPD], [test "x$with_libmicrohttpd" = xyes])
+
 # Check for plugin support
 AC_CANONICAL_HOST
 case "$host_os" in
@@ -628,4 +642,5 @@ AC_MSG_NOTICE([Summary of build options:
   Tests:              ${TESTS_INFO}
   Assertions:         $ENABLE_ASSERT
   POSIX xattr:        $ENABLE_XATTR
+  Microhttpd support: $with_libmicrohttpd
 ])

src/job.c

@@ -42,7 +42,8 @@
 
 void job_free(JOB *job)
 {
-	wget_http_free_challenges(&job->challenges);
+	if (job->challenges_alloc)
+		wget_http_free_challenges(&job->challenges);
 	wget_http_free_challenges(&job->proxy_challenges);
 	wget_metalink_free(&job->metalink);
 	wget_vector_free(&job->parts);

src/wget.c

@@ -507,8 +507,10 @@ static void add_url_to_queue(const char *url, wget_iri_t *base, const char *enco
 	if (config.spider || config.chunk_size)
 		new_job->head_first = 1;
 
-	if (config.auth_no_challenge)
+	if (config.auth_no_challenge) {
 		new_job->challenges = config.default_challenges;
+		new_job->challenges_alloc = false;
+	}
 
 	host_add_job(host, new_job);
 
@@ -1315,6 +1317,7 @@ static int process_response_header(wget_http_response_t *resp)
 		}
 
 		job->challenges = resp->challenges;
+		job->challenges_alloc = true;
 		resp->challenges = NULL;
 		job->inuse = 0; // try again, but with challenge responses
 		return 1; // stop further processing

src/wget_job.h

@@ -85,6 +85,8 @@ struct JOB {
 		auth_failure_count, // number of times server has returned a 401 response
 		mirror_pos, // where to look up the next (metalink) mirror to use
 		piece_pos; // where to look up the next (metalink) piece to download
+	bool
+		challenges_alloc; // Indicate whether the challenges vector is owned by the JOB
 	unsigned char
 		inuse : 1, // if job is already in use, 'used_by' holds the thread id of the downloader
 		sitemap : 1, // URL is a sitemap to be scanned in recursive mode