New release 0.1.5

.travis.yml

@@ -0,0 +1,9 @@
+language: c
+compiler:
+  - gcc
+  - clang
+# Change this to your needs
+script: ./autogen.sh && ./configure --enable-gtk-doc && make -j4 && make check -j4 && make distcheck -j4
+before_install:
+  - sudo apt-get -qq update
+  - sudo apt-get -q install autoconf automake autopoint libtool gtk-doc-tools gettext flex liblzma5 liblzma-dev libidn2-0 libidn2-0-dev libunistring0 libunistring-dev

ABOUT-NLS

@@ -0,0 +1 @@
+/usr/share/gettext/ABOUT-NLS

ChangeLog

@@ -1,3 +1,15 @@
+2014-02-03  gettextize  <bug-gnu-gettext@gnu.org>
+
+	* m4/gettext.m4: Upgrade to gettext-0.18.3.
+	* m4/iconv.m4: Upgrade to gettext-0.18.3.
+	* m4/lib-ld.m4: Upgrade to gettext-0.18.3.
+	* m4/lib-link.m4: Upgrade to gettext-0.18.3.
+	* m4/lib-prefix.m4: Upgrade to gettext-0.18.3.
+	* m4/nls.m4: Upgrade to gettext-0.18.3.
+	* m4/po.m4: Upgrade to gettext-0.18.3.
+	* m4/progtest.m4: Upgrade to gettext-0.18.3.
+	* configure.ac (AM_GNU_GETTEXT_VERSION): Bump to 0.18.3.
+
 2013-01-02  gettextize  <bug-gnu-gettext@gnu.org>
 
 	* m4/gettext.m4: Upgrade to gettext-0.18.1.

Makefile.am

@@ -1,5 +1,15 @@
-SUBDIRS = po include libmget src data examples tests docs/libmget
-ACLOCAL_AMFLAGS= -I m4
+# got some hints from https://gitorious.org/openismus-playground/examplelib/source
+
+SUBDIRS = po include libmget src data examples docs/libmget tests
+#ACLOCAL_AMFLAGS = -I m4
+ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS}
 
 # Enable GTK-Doc during make distcheck
-DISTCHECK_CONFIGURE_FLAGS=--enable-gtk-doc --enable-man
+DISTCHECK_CONFIGURE_FLAGS = --enable-gtk-doc --enable-man
+
+## Install the generated pkg-config file (.pc) into the expected location for
+## architecture-dependent package configuration information. Occasionally,
+## pkg-config files are also used for architecture-independent data packages,
+## in which case the correct install location would be $(datadir)/pkgconfig.
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = libmget-$(LIBMGET_API_VERSION).pc

NEWS

@@ -1,3 +1,31 @@
+26.02.2014
+  New release v0.1.5
+
+19.02.2014
+  Added support for RFC 6266 Content-Disposition
+
+17.02.2014
+  Fixed a race condition that could lead to sigfaults.
+  Fixed waiting for threads a program exit.
+
+16.02.2014
+  Made HTTPS test work with GnuTLS 2.12.
+  Added option -/t--tries to Mget.
+
+12.02.2014
+  Added HTTPS server code, used in test suite.
+  Added new test test-i-https.
+
+06.02.2014
+  Changed the naming convention of types.
+  Support pkg-config.
+  Fixed generating of man pages.
+
+01.02.2014
+  Added support for RFC 6797 HTTP Strict Transport Security (HSTS).
+  Do not download robots.txt twice for HTTP and HTTPS if --protocol-directories
+    is not given.
+
 24.01.2014
   New release v0.1.4
   Use libnettle for checksumming if libgnutls is not available or < 2.10.0.
@@ -12,7 +40,7 @@
 
 02.01.2014
   Added support for bzip2 Content-Encoding / Accept-Encoding type.
-    It easy going after having gzip and lzma decompression implemented.
+    It was easy going after having gzip and lzma decompression implemented.
   Created a mailing list for discussions around Mget.
     Subscribe by sending a mail to mget-bugs+subscribe@googlegroups.com.
 

README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/rockdaboot/mget.png?branch=develop)](https://travis-ci.org/rockdaboot/mget)
+
 Mget - multithreaded metalink / file / website downloader / spider and library
 ==============================================================================
 
@@ -46,6 +48,8 @@ Mget has already many features that go beyond what Wget provides.<br>
 
 An incomplete list of implemented features:
 
+- support for RFC 6266 compliant Content-Disposition
+- RFC 6797 HSTS (HTTP Strict Transport Security)
 - Support for bzip2 Content-Encoding / Accept-Encoding compression type
 - New Year gimmick: added support for XZ Content-Encoding / Accept-Encoding compression type
 - Character encoding of input files may be specified despite from local and remote encoding (--input-encoding)
@@ -102,9 +106,8 @@ Some ideas of what could be done next (but contact me before you start bigger ch
 - use [gtk-doc-tools](http://developer.gnome.org/gtk-doc-manual/unstable/settingup.html.en) for documentation.<br>
   I want the docs stay with the code: already tested Doxygen, but the man page support seems broken/orphaned.
 - WARC support
-- RFC 6797 HSTS (HTTP Strict Transport Security)
-  [Chromium HSTS domain list](https://src.chromium.org/viewvc/chrome/trunk/src/net/base/transport_security_state_static.json)
 - read credentials from secure wallets (e.g. kwallet, firefox, http://sourceforge.net/projects/passwordsafe/)
+  [Chromium HSTS domain list](http://src.chromium.org/viewvc/chrome/trunk/src/net/http/transport_security_state_static.json)
 - HTTP request pipelining
 - SPDY / HTTP2.0 protocol
 - respect data-urls

configure.ac

@@ -4,7 +4,7 @@ CFLAGS=$CFLAGS
 LDFLAGS=$LDFLAGS
 
 AC_PREREQ([2.67])
-AC_INIT([mget], [0.1.4], [mget-bugs@googlegroups.com])
+AC_INIT([mget], [0.1.5], [mget-bugs@googlegroups.com], [mget], [https://github.com/rockdaboot/mget])
 AM_INIT_AUTOMAKE
 
 CFLAGS=$CFLAGS
@@ -13,6 +13,12 @@ AC_USE_SYSTEM_EXTENSIONS
 LT_PREREQ([2.2])
 LT_INIT([dlopen])
 
+# Keep version info in configure.ac
+# see also "Updating library version information" at
+#   http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
+AC_SUBST([LIBMGET_SO_VERSION], [0:0:0])
+AC_SUBST([LIBMGET_API_VERSION], [0.1])
+
 AC_CONFIG_SRCDIR([src/mget.c])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
@@ -57,9 +63,9 @@ if test "x$EXTRA_CFLAGS" != "x" ; then
   CFLAGS="$CFLAGS $EXTRA_CFLAGS"
 fi
 
-dnl Check for large file support.  This check needs to come fairly
-dnl early because it could (in principle) affect whether functions and
-dnl headers are available, whether they work, etc.
+# Check for large file support.  This check needs to come fairly
+# early because it could (in principle) affect whether functions and
+# headers are available, whether they work, etc.
 AC_SYS_LARGEFILE
 AC_CHECK_SIZEOF(off_t)
 
@@ -197,10 +203,10 @@ AM_CONDITIONAL([WITH_BZIP2], [test $with_bzip2 = "yes"])
 AC_ARG_WITH(libidn2, AS_HELP_STRING([--without-libidn2], [disable IDN2 support]), with_libidn2=$withval, with_libidn2=yes)
 if test $with_libidn2 != "no"
 then
-  AC_CHECK_LIB(idn2, idn2_lookup_u8, [with_libidn2=yes; AC_SUBST(IDN_LIBS, "-lidn2") AC_DEFINE([WITH_LIBIDN2], [1], [Use libidn2])], [with_libidn2=no; AC_MSG_WARN(*** LIBIDN2 was not found. You will not able to use IDN2008 support)])
+  AC_CHECK_LIB(idn2, idn2_lookup_u8, [with_libidn2=yes; AC_SUBST(IDN_LIBS, "-lidn2") AC_DEFINE([WITH_LIBIDN2], [1], [Use libidn2])], [with_libidn2=no; AC_MSG_WARN(*** LIBIDN2 was not found. You will not be able to use IDN2008 support)])
   if test x$with_libidn2 = "xyes"
   then
-    AC_CHECK_LIB(unistring, u8_tolower, [with_libunistring=yes; AC_SUBST(IDN_LIBS, "-lidn2 -lunistring") AC_DEFINE([WITH_LIBUNISTRING], [1], [Use libunistring])], [with_libunistring=no; AC_MSG_WARN(*** LIBUNISTRING was not found. You will not able to use IDN2008 on uppercase domains)])
+    AC_CHECK_LIB(unistring, u8_tolower, [with_libunistring=yes; AC_SUBST(IDN_LIBS, "-lidn2 -lunistring") AC_DEFINE([WITH_LIBUNISTRING], [1], [Use libunistring])], [with_libunistring=no; AC_MSG_WARN(*** LIBUNISTRING was not found. You will not be able to use IDN2008 on uppercase domains)])
   fi
 fi
 AM_CONDITIONAL([WITH_LIBUNISTRING], [test x$with_libunistring = "xyes"])
@@ -264,7 +270,8 @@ AC_CONFIG_FILES([Makefile
                  docs/libmget/Makefile
                  data/Makefile
                  examples/Makefile
-                 tests/Makefile])
+                 tests/Makefile
+                 libmget-${LIBMGET_API_VERSION}.pc:libmget.pc.in])
 AC_OUTPUT
 
 AC_MSG_NOTICE([Summary of build options:

docs/libmget/Makefile.am

@@ -29,11 +29,11 @@ DOC_SOURCE_DIR=$(top_srcdir)/libmget $(top_srcdir)/include
 
 # Extra options to supply to gtkdoc-scan.
 # e.g. SCAN_OPTIONS=--deprecated-guards="GTK_DISABLE_DEPRECATED"
-SCAN_OPTIONS=
+SCAN_OPTIONS=--ignore-decorators="G_GNUC_MGET_UNUSED"
 
 # Extra options to supply to gtkdoc-mkdb.
 # e.g. MKDB_OPTIONS=--xml-mode --output-format=xml
-MKDB_OPTIONS=--xml-mode --output-format=xml
+MKDB_OPTIONS=--xml-mode --output-format=xml --ignore-files=css_tokenizer.c
 
 # Extra options to supply to gtkdoc-mktmpl
 # e.g. MKTMPL_OPTIONS=--only-section-tmpl
@@ -104,31 +104,29 @@ endif
 
 -include $(top_srcdir)/git.mk
 
+theMANS =
+man_MANS =
+
 if ENABLE_GTK_DOC
 if ENABLE_MAN
 
-#man_MANS = libmget.3
-man_MANS = \
- libmget_list_routines.3 \
- libmget_memory_allocation_routines.3
-
-#libmget.3: $(srcdir)/xml/libmget.xml
-#libmget_list_routines.3: $(srcdir)/xml/libmget.xml
-#	@XSLTPROC@ -nonet http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl $<
-libmget_list_routines.3: sgml.stamp
-	@XSLTPROC@ -nonet http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl xml/libmget-list.xml
+theMANS += libmget-xalloc.3 libmget-utils.3 libmget-list.3
+man_MANS += docs $(theMANS)
 
-libmget_memory_allocation_routines.3: sgml.stamp
-	@XSLTPROC@ -nonet http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl xml/libmget-xalloc.xml
+%.3:
+#.xml.3:
+	@file=xml/`echo $@|cut -d'.' -f1`.xml; \
+	@XSLTPROC@ -nonet http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl $$file
 
 endif
 endif
 
-BUILT_EXTRA_DIST = $(man_MANS)
-#EXTRA_DIST += $(man_MANS)
-CLEANFILES += $(man_MANS)
+BUILT_EXTRA_DIST = $(theMANS)
+EXTRA_DIST += $(theMANS)
+CLEANFILES ?=
+CLEANFILES += $(theMANS) libmget-overrides.txt libmget-decl.txt libmget-decl-list.txt
 
-#dist-hook-local: all-local
+dist-hook-local: all-local
 
-#mgetlib-docs-clean: clean
-#	cd $(srcdir) && rm -rf xml html
+libmget-docs-clean: clean
+	cd $(srcdir) && rm -rf xml html

docs/libmget/libmget-docs.sgml

@@ -16,8 +16,10 @@
 
   <chapter>
     <title>LibMget functions</title>
-        <xi:include href="xml/libmget-list.xml"/>
         <xi:include href="xml/libmget-xalloc.xml"/>
+        <xi:include href="xml/libmget-list.xml"/>
+        <xi:include href="xml/libmget-utils.xml"/>
+        <xi:include href="xml/libmget-io.xml"/>
   </chapter>
   <chapter id="object-tree">
     <title>Object Hierarchy</title>
@@ -32,5 +34,7 @@
     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
   </index>
 
+<!--
   <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
+-->
 </book>

docs/libmget/libmget-sections.txt

@@ -1,7 +1,27 @@
+<SECTION>
+<FILE>libmget-xalloc</FILE>
+<TITLE>Memory Allocation Functions</TITLE>
+mget_set_oomfunc
+mget_malloc
+mget_calloc
+mget_realloc
+</SECTION>
+
+<SECTION>
+<FILE>libmget-utils</FILE>
+<TITLE>General utility functions</TITLE>
+mget_strcmp
+mget_strcasecmp
+mget_strncmp
+mget_strncasecmp
+mget_memtohex
+mget_millisleep
+</SECTION>
+
 <SECTION>
 <FILE>libmget-list</FILE>
 <TITLE>Double Linked List</TITLE>
-MGET_LIST
+mget_list_t
 mget_list_append
 mget_list_prepend
 mget_list_getfirst
@@ -12,10 +32,11 @@ mget_list_free
 </SECTION>
 
 <SECTION>
-<FILE>libmget-xalloc</FILE>
-<TITLE>Memory Allocation Functions</TITLE>
-mget_set_oomfunc
-mget_malloc
-mget_calloc
-mget_realloc
+<FILE>libmget-io</FILE>
+<TITLE>IO helper routines</TITLE>
+mget_fdgetline
+mget_getline
+mget_ready_2_read
+mget_ready_2_write
+mget_read_file
 </SECTION>

examples/Makefile.am

@@ -3,7 +3,7 @@ noinst_PROGRAMS = print_css_urls print_css_urls2 print_css_urls3 http_get\
 
 AM_CPPFLAGS = -I$(top_srcdir)/include
 AM_LDFLAGS = -static
-LDADD = ../libmget/libmget.la
+LDADD = ../libmget/libmget-@LIBMGET_API_VERSION@.la
 # local_ldadd = $(top_srcdir)/libmget/libmget.la
 
 #print_css_urls_SOURCES = print_css_urls.c

examples/getstream.c

@@ -45,7 +45,7 @@ static int metaint, streamdatalen, metadatalen;
 // callback function to examine received HTTP response header
 // <context> depends on MGET_HTTP_BODY_SAVEAS_* option given to mget_http_get().
 // The response header is has been parsed into <resp> structure.
-static void header_handler(void *context G_GNUC_MGET_UNUSED, MGET_HTTP_RESPONSE *resp)
+static void header_callback(void *context G_GNUC_MGET_UNUSED, mget_http_response_t *resp)
 {
 	// If you are looking for header that are ignored by libmget, parse them yourself.
 
@@ -68,7 +68,7 @@ static void header_handler(void *context G_GNUC_MGET_UNUSED, MGET_HTTP_RESPONSE
 }
 
 // callback function to handle incoming stream data
-static void stream_handler(void *context G_GNUC_MGET_UNUSED, const char *data, size_t len)
+static void stream_callback(void *context G_GNUC_MGET_UNUSED, const char *data, size_t len)
 {
 	// any stream data received is piped through this function
 
@@ -103,7 +103,7 @@ static void stream_handler(void *context G_GNUC_MGET_UNUSED, const char *data, s
 
 int main(int argc G_GNUC_MGET_UNUSED, const char *const *argv G_GNUC_MGET_UNUSED)
 {
-	MGET_HTTP_RESPONSE *resp;
+	mget_http_response_t *resp;
 	char *stream_url = NULL;
 
 	// set up libmget global configuration
@@ -124,7 +124,7 @@ int main(int argc G_GNUC_MGET_UNUSED, const char *const *argv G_GNUC_MGET_UNUSED
 	}
 
 	// free the response
-	http_free_response(&resp);
+	mget_http_free_response(&resp);
 
 	// The icy-metaint: response header indicates the <size> of the data blocks.
 	// The stream starts with <size> data bytes followed by one single byte, that holds the size of the metadata divided by 16.
@@ -134,12 +134,12 @@ int main(int argc G_GNUC_MGET_UNUSED, const char *const *argv G_GNUC_MGET_UNUSED
 		resp = mget_http_get(
 			MGET_HTTP_URL, stream_url,
 			MGET_HTTP_HEADER_ADD, "Icy-Metadata: 1", // we want in-stream title/actor information
-			MGET_HTTP_HEADER_FUNC, header_handler, // callback used to parse special headers like 'Icy-Name'
+			MGET_HTTP_HEADER_FUNC, header_callback, // callback used to parse special headers like 'Icy-Name'
 			// MGET_HTTP_HEADER_SAVEAS_STREAM, stdout,
-			MGET_HTTP_BODY_SAVEAS_FUNC, stream_handler, // callback to cut title info out of audio stream
+			MGET_HTTP_BODY_SAVEAS_FUNC, stream_callback, // callback to cut title info out of audio stream
 			NULL);
 
-		http_free_response(&resp);
+		mget_http_free_response(&resp);
 	}
 
 	// free resources - needed for valgrind testing

examples/http_get.c

@@ -35,8 +35,8 @@
 
 int main(int argc G_GNUC_MGET_UNUSED, const char *const *argv G_GNUC_MGET_UNUSED)
 {
-	MGET_HTTP_CONNECTION *conn = NULL;
-	MGET_HTTP_RESPONSE *resp;
+	mget_http_connection_t *conn = NULL;
+	mget_http_response_t *resp;
 
 	// set up libmget global configuration
 	mget_global_init(
@@ -46,7 +46,7 @@ int main(int argc G_GNUC_MGET_UNUSED, const char *const *argv G_GNUC_MGET_UNUSED
 		MGET_DNS_CACHING, 1,
 		MGET_COOKIES_ENABLED, 1,
 		MGET_COOKIE_SUFFIXES, "public_suffixes.txt",
-		MGET_COOKIE_STORE, "cookies.txt",
+		MGET_COOKIE_FILE, "cookies.txt",
 		MGET_COOKIE_KEEPSESSIONCOOKIES, 1,
 		// MGET_BIND_ADDRESS, "127.0.0.1:6666",
 		// MGET_NET_FAMILY_EXCLUSIVE, MGET_NET_FAMILY_IPV4, // or MGET_NET_FAMILY_IPV6 or MGET_NET_FAMILY_ANY
@@ -74,11 +74,11 @@ int main(int argc G_GNUC_MGET_UNUSED, const char *const *argv G_GNUC_MGET_UNUSED
 		printf("%s%s\n", resp->header->data, resp->body->data);
 
 		// free the response
-		http_free_response(&resp);
+		mget_http_free_response(&resp);
 	}
 
 	// close connection if still open
-	http_close(&conn);
+	mget_http_close(&conn);
 
 	// free resources - needed for valgrind testing
 	mget_global_deinit();