From ad05540395b0a03cc0468cf3f66015e23ac613a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Pawe=C5=82czyk?= Date: Thu, 7 Sep 2023 11:36:14 +0200 Subject: [PATCH] build : do not use _GNU_SOURCE gratuitously (#1129) * Do not use _GNU_SOURCE gratuitously. What is needed to build whisper.cpp and examples is availability of stuff defined in The Open Group Base Specifications Issue 6 (https://pubs.opengroup.org/onlinepubs/009695399/) known also as Single Unix Specification v3 (SUSv3) or POSIX.1-2001 + XSI extensions, plus some stuff from BSD that is not specified in POSIX.1. Well, that was true until NUMA support was added recently in ggml, so enable GNU libc extensions for Linux builds to cover that. There is no need to penalize musl libc which simply follows standards. Not having feature test macros in source code gives greater flexibility to those wanting to reuse it in 3rd party app, as they can build it with minimal FTM (_XOPEN_SOURCE=600) or other FTM depending on their needs. It builds without issues in Alpine (musl libc), Ubuntu (glibc), MSYS2. * examples : include SDL headers before other headers Avoid macOS build error when _DARWIN_C_SOURCE is not defined, brought by SDL2 relying on Darwin extension memset_pattern4/8/16 (from string.h). * make : enable BSD extensions for DragonFlyBSD to expose RLIMIT_MEMLOCK * make : use BSD-specific FTMs to enable alloca on BSDs * make : fix OpenBSD build by exposing newer POSIX definitions * cmake : follow recent FTM improvements from Makefile --- CMakeLists.txt | 43 ++++++++++++++++++++++++++++ Makefile | 45 +++++++++++++++++++++++++++--- examples/command/command.cpp | 2 +- examples/stream/stream.cpp | 2 +- examples/talk-llama/llama.cpp | 8 ------ examples/talk-llama/talk-llama.cpp | 2 +- examples/talk/talk.cpp | 2 +- ggml.c | 1 - 8 files changed, 88 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 407d9800ca7..882d3992221 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -321,6 +321,49 @@ else() endif() endif() +# clock_gettime came in POSIX.1b (1993) +# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional +# posix_memalign came in POSIX.1-2001 / SUSv3 +# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985) +add_compile_definitions(_XOPEN_SOURCE=600) + +# Somehow in OpenBSD whenever POSIX conformance is specified +# some string functions rely on locale_t availability, +# which was introduced in POSIX.1-2008, forcing us to go higher +IF (CMAKE_SYSTEM_NAME MATCHES "OpenBSD") + remove_definitions(-D_XOPEN_SOURCE=600) + add_compile_definitions(_XOPEN_SOURCE=700) +ENDIF() + +# Data types, macros and functions related to controlling CPU affinity +# are available on Linux through GNU extensions in libc +IF (CMAKE_SYSTEM_NAME MATCHES "Linux") + add_compile_definitions(_GNU_SOURCE) +ENDIF() + +# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1, +# and on macOS its availability depends on enabling Darwin extensions +# similarly on DragonFly, enabling BSD extensions is necessary +IF (CMAKE_SYSTEM_NAME MATCHES "Darwin") + add_compile_definitions(_DARWIN_C_SOURCE) +ENDIF() +IF (CMAKE_SYSTEM_NAME MATCHES "DragonFly") + add_compile_definitions(_DARWIN_C_SOURCE) +ENDIF() + +# alloca is a non-standard interface that is not visible on BSDs when +# POSIX conformance is specified, but not all of them provide a clean way +# to enable it in such cases +IF (CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + add_compile_definitions(__BSD_VISIBLE) +ENDIF() +IF (CMAKE_SYSTEM_NAME MATCHES "NetBSD") + add_compile_definitions(_NETBSD_SOURCE) +ENDIF() +IF (CMAKE_SYSTEM_NAME MATCHES "OpenBSD") + add_compile_definitions(_BSD_SOURCE) +ENDIF() + if (WHISPER_PERF) set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DGGML_PERF) endif() diff --git a/Makefile b/Makefile index 066ad59804d..ecbbcff0def 100644 --- a/Makefile +++ b/Makefile @@ -42,18 +42,55 @@ CFLAGS = -I. -O3 -DNDEBUG -std=c11 -fPIC CXXFLAGS = -I. -I./examples -O3 -DNDEBUG -std=c++11 -fPIC LDFLAGS = -# ref: https://github.com/ggerganov/whisper.cpp/issues/37 -ifneq ($(wildcard /usr/include/musl/*),) - CFLAGS += -D_POSIX_SOURCE -D_GNU_SOURCE - CXXFLAGS += -D_POSIX_SOURCE -D_GNU_SOURCE +# clock_gettime came in POSIX.1b (1993) +# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional +# posix_memalign came in POSIX.1-2001 / SUSv3 +# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985) +CFLAGS += -D_XOPEN_SOURCE=600 +CXXFLAGS += -D_XOPEN_SOURCE=600 + +# Somehow in OpenBSD whenever POSIX conformance is specified +# some string functions rely on locale_t availability, +# which was introduced in POSIX.1-2008, forcing us to go higher +ifeq ($(UNAME_S),OpenBSD) + CFLAGS += -U_XOPEN_SOURCE -D_XOPEN_SOURCE=700 + CXXFLAGS += -U_XOPEN_SOURCE -D_XOPEN_SOURCE=700 +endif + +# Data types, macros and functions related to controlling CPU affinity +# are available on Linux through GNU extensions in libc +ifeq ($(UNAME_S),Linux) + CFLAGS += -D_GNU_SOURCE + CXXFLAGS += -D_GNU_SOURCE endif # RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1, # and on macOS its availability depends on enabling Darwin extensions +# similarly on DragonFly, enabling BSD extensions is necessary ifeq ($(UNAME_S),Darwin) CFLAGS += -D_DARWIN_C_SOURCE CXXFLAGS += -D_DARWIN_C_SOURCE endif +ifeq ($(UNAME_S),DragonFly) + CFLAGS += -D__BSD_VISIBLE + CXXFLAGS += -D__BSD_VISIBLE +endif + +# alloca is a non-standard interface that is not visible on BSDs when +# POSIX conformance is specified, but not all of them provide a clean way +# to enable it in such cases +ifeq ($(UNAME_S),FreeBSD) + CFLAGS += -D__BSD_VISIBLE + CXXFLAGS += -D__BSD_VISIBLE +endif +ifeq ($(UNAME_S),NetBSD) + CFLAGS += -D_NETBSD_SOURCE + CXXFLAGS += -D_NETBSD_SOURCE +endif +ifeq ($(UNAME_S),OpenBSD) + CFLAGS += -D_BSD_SOURCE + CXXFLAGS += -D_BSD_SOURCE +endif # OS specific # TODO: support Windows diff --git a/examples/command/command.cpp b/examples/command/command.cpp index 54e3549f3bc..d39af7309a2 100644 --- a/examples/command/command.cpp +++ b/examples/command/command.cpp @@ -6,8 +6,8 @@ // ref: https://github.com/ggerganov/whisper.cpp/issues/171 // -#include "common.h" #include "common-sdl.h" +#include "common.h" #include "whisper.h" #include diff --git a/examples/stream/stream.cpp b/examples/stream/stream.cpp index 4c7f7d1af47..7f869d8d0a2 100644 --- a/examples/stream/stream.cpp +++ b/examples/stream/stream.cpp @@ -3,8 +3,8 @@ // A very quick-n-dirty implementation serving mainly as a proof of concept. // -#include "common.h" #include "common-sdl.h" +#include "common.h" #include "whisper.h" #include diff --git a/examples/talk-llama/llama.cpp b/examples/talk-llama/llama.cpp index aecae009d05..30fd444027e 100644 --- a/examples/talk-llama/llama.cpp +++ b/examples/talk-llama/llama.cpp @@ -1,11 +1,3 @@ -// Defines fileno on msys: -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#include -#include -#include -#endif - #include "llama-util.h" #include "llama.h" diff --git a/examples/talk-llama/talk-llama.cpp b/examples/talk-llama/talk-llama.cpp index 61d8583d24c..0a9b00c971f 100644 --- a/examples/talk-llama/talk-llama.cpp +++ b/examples/talk-llama/talk-llama.cpp @@ -1,8 +1,8 @@ // Talk with AI // -#include "common.h" #include "common-sdl.h" +#include "common.h" #include "whisper.h" #include "llama.h" diff --git a/examples/talk/talk.cpp b/examples/talk/talk.cpp index 85c103dae87..346d9d483fe 100644 --- a/examples/talk/talk.cpp +++ b/examples/talk/talk.cpp @@ -1,8 +1,8 @@ // Talk with AI // -#include "common.h" #include "common-sdl.h" +#include "common.h" #include "whisper.h" #include "gpt-2.h" diff --git a/ggml.c b/ggml.c index eac00bbd672..0fe07b245ba 100644 --- a/ggml.c +++ b/ggml.c @@ -1,4 +1,3 @@ -#define _GNU_SOURCE // Defines CLOCK_MONOTONIC on Linux #define _CRT_SECURE_NO_DEPRECATE // Disables ridiculous "unsafe" warnigns on Windows #include "ggml.h"