Add MinGW-w64 builds (@ed-alertedh, #2553)

* Add build configuration for MinGW-w64. Primarily just needed to use _WIN32 instead of _MSC_VER in header guards. Various other small patches included to account for compiler differences.

* try alternative win32 datetime format method avoiding ATL (which isn't available in MinGW)

.travis.yml

@@ -13,6 +13,7 @@ addons:
     # required by openssl installer
     - perl
 
+
 matrix:
  include:
  - name: "Linux GCC: +centos +debian"
@@ -48,6 +49,14 @@ matrix:
    env: LINKAGE=static HOMEBREW_NO_AUTO_UPDATE=1
    before_script:
     - ./configure --install-deps --disable-lz4-ext --prefix="$PWD/dest" --enable-static --disable-syslog
+ - name: "Windows MinGW-w64"
+   os: windows
+   env:
+    - SKIP_MAKE=y
+   before_install:
+    - source ./packaging/mingw-w64/travis-before-install.sh
+   before_script:
+    - ./packaging/mingw-w64/configure-build-msys2-mingw.sh
  - name: "Linux GCC: +integration-tests +copyright-check +doc-check"
    os: linux
    dist: xenial
@@ -74,11 +83,14 @@ install:
  - rm -rf artifacts dest
  - mkdir dest artifacts
 
+before_cache:
+ - if [[ $TRAVIS_OS_NAME == windows ]]; then $msys2 pacman --sync --clean --noconfirm ; fi
+
 before_script:
 script:
-- make -j2 all examples check && make -j2 -C tests build
-- if [[ $RUN_INTEGRATION_TESTS != y ]]; then make -C tests run_local_quick ; fi
-- make install
+- if [[ $SKIP_MAKE != y ]]; then make -j2 all examples check && make -j2 -C tests build ; fi
+- if [[ $SKIP_MAKE != y && $RUN_INTEGRATION_TESTS != y ]]; then make -C tests run_local_quick ; fi
+- if [[ $SKIP_MAKE != y ]]; then make install ; fi
 - if [[ -z $NO_ARTIFACTS ]]; then (cd dest && tar cvzf ../artifacts/librdkafka-${CC}.tar.gz .) ; fi
 - for distro in $ADDITIONAL_BUILDS ; do packaging/tools/distro-build.sh $distro || exit 1 ; done
 - if [[ $COPYRIGHT_CHECK == y ]]; then make copyright-check ; fi

examples/kafkatest_verifiable_client.cpp

@@ -46,7 +46,7 @@
 #include <ctype.h>
 #include <strings.h>
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 #include "../win32/wingetopt.h"
 #elif _AIX
 #include <unistd.h>

examples/rdkafka_complex_consumer_example.cpp

@@ -39,13 +39,14 @@
 #include <csignal>
 #include <cstring>
 
-#ifndef _MSC_VER
+#ifndef _WIN32
 #include <sys/time.h>
+#else
+#include <windows.h> /* for GetLocalTime */
 #endif
 
 #ifdef _MSC_VER
 #include "../win32/wingetopt.h"
-#include <atltime.h>
 #elif _AIX
 #include <unistd.h>
 #else
@@ -77,15 +78,19 @@ static void sigterm (int sig) {
  * @brief format a string timestamp from the current time
  */
 static void print_time () {
-#ifndef _MSC_VER
+#ifndef _WIN32
         struct timeval tv;
         char buf[64];
         gettimeofday(&tv, NULL);
         strftime(buf, sizeof(buf) - 1, "%Y-%m-%d %H:%M:%S", localtime(&tv.tv_sec));
         fprintf(stderr, "%s.%03d: ", buf, (int)(tv.tv_usec / 1000));
 #else
-        std::wcerr << CTime::GetCurrentTime().Format(_T("%Y-%m-%d %H:%M:%S")).GetString()
-                << ": ";
+        SYSTEMTIME lt = {0};
+        GetLocalTime(&lt);
+        // %Y-%m-%d %H:%M:%S.xxx:
+        fprintf(stderr, "%04d-%02d-%02d %02d:%02d:%02d.%03d: ",
+            lt.wYear, lt.wMonth, lt.wDay,
+            lt.wHour, lt.wMinute, lt.wSecond, lt.wMilliseconds);
 #endif
 }
 class ExampleEventCb : public RdKafka::EventCb {
@@ -433,7 +438,7 @@ int main (int argc, char **argv) {
     delete msg;
   }
 
-#ifndef _MSC_VER
+#ifndef _WIN32
   alarm(10);
 #endif
 

examples/rdkafka_consume_batch.cpp

@@ -44,11 +44,11 @@
 #include <csignal>
 #include <cstring>
 
-#ifndef _MSC_VER
+#ifndef _WIN32
 #include <sys/time.h>
 #endif
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 #include "../win32/wingetopt.h"
 #include <atltime.h>
 #elif _AIX
@@ -78,7 +78,7 @@ static void sigterm (int sig) {
  * @returns the current wall-clock time in milliseconds
  */
 static int64_t now () {
-#ifndef _MSC_VER
+#ifndef _WIN32
         struct timeval tv;
         gettimeofday(&tv, NULL);
         return ((int64_t)tv.tv_sec * 1000) + (tv.tv_usec / 1000);

examples/rdkafka_example.cpp

@@ -39,7 +39,7 @@
 #include <csignal>
 #include <cstring>
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 #include "../win32/wingetopt.h"
 #elif _AIX
 #include <unistd.h>

examples/rdkafka_performance.c

@@ -52,7 +52,7 @@
 #include "rd.h"
 #include "rdtime.h"
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 #include "../win32/wingetopt.h"
 #include "../win32/wintime.h"
 #endif
@@ -774,7 +774,7 @@ static rd_kafka_resp_err_t do_produce (rd_kafka_t *rk,
  */
 static void do_sleep (int sleep_us) {
         if (sleep_us > 100) {
-#ifdef _MSC_VER
+#ifdef _WIN32
                 Sleep(sleep_us / 1000);
 #else
                 usleep(sleep_us);
@@ -1263,7 +1263,7 @@ int main (int argc, char **argv) {
         if (stats_intvlstr) {
                 /* User enabled stats (-T) */
 
-#ifndef _MSC_VER
+#ifndef _WIN32
                 if (stats_cmd) {
                         if (!(stats_fp = popen(stats_cmd,
 #ifdef __linux__
@@ -1696,7 +1696,7 @@ int main (int argc, char **argv) {
 		fclose(latency_fp);
 
         if (stats_fp) {
-#ifndef _MSC_VER
+#ifndef _WIN32
                 pclose(stats_fp);
 #endif
                 stats_fp = NULL;

packaging/mingw-w64/configure-build-msys2-mingw.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+$mingw64 cmake -DCMAKE_MAKE_PROGRAM=mingw32-make -G "MinGW Makefiles" \
+      -DCMAKE_INSTALL_PREFIX=$PWD/dest/ \
+      -DMINGW_BUILD:BOOL=ON \
+      -DWITHOUT_WIN32_CONFIG:BOOL=ON \
+      -DRDKAFKA_BUILD_EXAMPLES:BOOL=ON \
+      -DRDKAFKA_BUILD_TESTS:BOOL=ON \
+      -DWITH_LIBDL:BOOL=OFF \
+      -DWITH_PLUGINS:BOOL=OFF \
+      -DWITH_SASL:BOOL=ON \
+      -DWITH_SSL:BOOL=ON \
+      -DWITH_ZLIB:BOOL=OFF \
+      -DRDKAFKA_BUILD_STATIC:BOOL=OFF \
+      -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS:BOOL=TRUE .
+
+$mingw64 mingw32-make
+$mingw64 mingw32-make install
+
+export PATH="$PWD/dest/bin:/mingw64/bin/:${PATH}"
+cd tests
+./test-runner.exe -l -Q -p1

packaging/mingw-w64/travis-before-install.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+# Slightly modified from:
+# https://docs.travis-ci.com/user/reference/windows/#how-do-i-use-msys2
+case $TRAVIS_OS_NAME in
+    windows)
+        [[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64
+        choco uninstall -y mingw
+        choco upgrade --no-progress -y msys2
+        export msys2='cmd //C RefreshEnv.cmd '
+        export msys2+='& set MSYS=winsymlinks:nativestrict '
+        export msys2+='& C:\\tools\\msys64\\msys2_shell.cmd -defterm -no-start'
+        export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --"
+        export msys2+=" -msys2 -c "\"\$@"\" --"
+        $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-cyrus-sasl
+        ## Install more MSYS2 packages from https://packages.msys2.org/base here
+        taskkill //IM gpg-agent.exe //F  || true  # https://travis-ci.community/t/4967
+        export PATH=/C/tools/msys64/mingw64/bin:$PATH
+        export MAKE=mingw32-make  # so that Autotools can find it
+        ;;
+esac

src-cpp/HandleImpl.cpp

@@ -295,7 +295,7 @@ void RdKafka::HandleImpl::set_common_config (RdKafka::ConfImpl *confimpl) {
   }
 
   if (confimpl->open_cb_) {
-#ifndef _MSC_VER
+#ifndef _WIN32
     rd_kafka_conf_set_open_cb(confimpl->rk_conf_, RdKafka::open_cb_trampoline);
     open_cb_ = confimpl->open_cb_;
 #endif

src-cpp/rdkafkacpp.h

@@ -55,7 +55,7 @@
 #include <stdint.h>
 #include <sys/types.h>
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 #ifndef ssize_t
 #ifndef _BASETSD_H_
 #include <basetsd.h>

src-cpp/rdkafkacpp_int.h

@@ -40,7 +40,7 @@ extern "C" {
 #include "../src/rdkafka.h"
 }
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 /* Visual Studio */
 #include "../src/win32_config.h"
 #else

src/CMakeLists.txt

@@ -167,13 +167,23 @@ if(NOT RDKAFKA_BUILD_STATIC)
   set_property(TARGET rdkafka PROPERTY SOVERSION ${LIBVER})
 endif()
 
+if(MINGW_BUILD)
+  # Target Windows 8.1 to match the VS projects (MinGW defaults to an older WinAPI version)
+  list(APPEND rdkafka_compile_definitions WINVER=0x0603 _WIN32_WINNT=0x0603 UNICODE)
+endif(MINGW_BUILD)
+
 # Support '#include <rdkafka.h>'
 target_include_directories(rdkafka PUBLIC "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}>")
 target_compile_definitions(rdkafka PUBLIC ${rdkafka_compile_definitions})
 if(RDKAFKA_BUILD_STATIC)
   target_compile_definitions(rdkafka PUBLIC LIBRDKAFKA_STATICLIB)
 endif()
 
+if(MINGW_BUILD)
+  # MinGW ignores `#pragma comment(lib, ...)` so we need to specify link libs
+  target_link_libraries(rdkafka PUBLIC crypt32 ws2_32 secur32)
+endif(MINGW_BUILD)
+
 if(WIN32)  
   if(RDKAFKA_BUILD_STATIC)
     target_link_libraries(rdkafka PUBLIC crypt32)

src/crc32c.c

@@ -50,7 +50,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
-#ifndef _MSC_VER
+#ifndef _WIN32
 #include <unistd.h>
 #endif
 

src/rd.h

@@ -30,7 +30,7 @@
 #ifndef _RD_H_
 #define _RD_H_
 
-#ifndef _MSC_VER
+#ifndef _WIN32
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE  /* for strndup() */
 #endif
@@ -56,15 +56,15 @@
 #include "tinycthread.h"
 #include "rdsysqueue.h"
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 /* Visual Studio */
 #include "win32_config.h"
 #else
 /* POSIX / UNIX based systems */
 #include "../config.h" /* mklove output */
 #endif
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 /* Win32/Visual Studio */
 #include "rdwin32.h"
 
@@ -131,7 +131,7 @@ static RD_INLINE RD_UNUSED void rd_free(void *ptr) {
 }
 
 static RD_INLINE RD_UNUSED char *rd_strdup(const char *s) {
-#ifndef _MSC_VER
+#ifndef _WIN32
 	char *n = strdup(s);
 #else
 	char *n = _strdup(s);
@@ -186,7 +186,7 @@ static RD_INLINE RD_UNUSED char *rd_strndup(const char *s, size_t len) {
 #ifdef __APPLE__
 /* Some versions of MacOSX dont have IOV_MAX */
 #define IOV_MAX 1024
-#elif defined(_MSC_VER) || defined(__GNU__)
+#elif defined(_WIN32) || defined(__GNU__)
 /* There is no IOV_MAX on MSVC or GNU but it is used internally in librdkafka */
 #define IOV_MAX 1024
 #else

src/rdaddr.c

@@ -27,12 +27,13 @@
  */
 
 
+
 #include "rd.h"
 #include "rdaddr.h"
 #include "rdrand.h"
 
-#ifdef _MSC_VER
-#include <WS2tcpip.h>
+#ifdef _WIN32
+#include <ws2tcpip.h>
 #endif
 
 const char *rd_sockaddr2str (const void *addr, int flags) {
@@ -173,7 +174,7 @@ rd_sockaddr_list_t *rd_getaddrinfo (const char *nodesvc, const char *defsvc,
 #endif
 			*errstr = rd_strerror(errno);
 		else {
-#ifdef _MSC_VER
+#ifdef _WIN32
 			*errstr = gai_strerrorA(r);
 #else
 			*errstr = gai_strerror(r);

src/rdaddr.h

@@ -29,13 +29,13 @@
 #ifndef _RDADDR_H_
 #define _RDADDR_H_
 
-#ifndef _MSC_VER
+#ifndef _WIN32
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <netdb.h>
 #else
 #define WIN32_MEAN_AND_LEAN
-#include <WinSock2.h>
+#include <winsock2.h>
 #include <ws2ipdef.h>
 #endif