Permalink
Browse files

CMake build definitions (#1624)

* WIP cmake files

* Cmake changes

* WIP changes

* Add doc makefile

* Tests are building and running

* Add help2man, fix install

* Fix build issue for vwdll

* Get Windows cmake build working, fix debug/release configurations

* Fix quotes

* Install now works similar to automake, pkgconfig changes for cmake

* Fix test issue

* Add warnings option

* Add TODO and update comment

* Simplify targets

* Convert tests to proper cmake test targets:

* Remove all code references to automake

* Fix versioning workflow across windows and linux

* Add install for java to cmake, fix version parsing

* - Update build linux script for cmake
- Add cmake support for Java and Python
- Rewrite most of Python setup file for CMake

* Remove makefiles

* Don't make Java required

* Update org name in readme, add vw-bin to test deps

* Fix Python version selection for package build, fix test PATH for vw finding

* Test if earlier cmake version works

* Upgrade cmake in build

* Fix apt interactive issue

* Show commands for debugging

* Cd to correct directory

* Add override for Travis because gcc 4.9 does not support codecvt

* Make Java target directory

* Link for gcov

* Change coverage test run

* Add static linked binary option

* Update readme with cmake instructions. Change base makefile to be a cmake helper script for education

* Fix python build definition for windows and linux

* Fix lgtm warnings introduced in this PR

* Add some gotchas

* Changes for MacOS, add Python libs, update README

* Fix static linking, makefile targets

* Fix some lgtm issues

* Can turn test target on or off

* Fix source dir variable for explore
  • Loading branch information...
jackgerrits authored and lokitoth committed Nov 5, 2018
1 parent fb8d488 commit 2a8f4975904f172acbfabccc85890128a8b58bd9
@@ -16,7 +16,6 @@ libtool
ltmain.sh
missing
vowpalwabbit/Makefile.in
vowpalwabbit/config.h.in
config.h
*.la
*.lo
@@ -128,3 +127,5 @@ index.html
packages-microsoft-prod.deb
reinforcement_learning/unit_test/rlclient-test.out
cs/azure_service/azure_service.ccproj.user

build
@@ -0,0 +1,111 @@
cmake_minimum_required(VERSION 3.5)

# Only allow Release and Debug configurations
set(CMAKE_CONFIGURATION_TYPES Debug Release CACHE TYPE INTERNAL FORCE)

project(vowpal_wabbit C CXX)
set(CMAKE_CXX_STANDARD 11)

# VW targets Windows 8.1 SDK
if(WIN32)
set(CMAKE_SYSTEM_VERSION 8.1 CACHE TYPE INTERNAL FORCE)
endif()

# Read version into variable
file(READ version.txt PACKAGE_VERSION)
string(STRIP ${PACKAGE_VERSION} PACKAGE_VERSION)
message("Version: ${PACKAGE_VERSION}")

include(ProcessorCount)
ProcessorCount(NumProcessors)
message("Number of processors: ${NumProcessors}")
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/nprocs.txt ${NumProcessors})

set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of build, options are: Debug, Release" FORCE)

option(PROFILE "Turn on flags required for profiling" OFF)
option(VALGRIND_PROFILE "Turn on flags required for profiling with valgrind" OFF)
option(GCOV "Turn on flags required for gcov" OFF)
option(WARNINGS "Turn on warning flags. ON by default." ON)
option(STATIC_LINK_VW "Link VW executable statically. Off by default." OFF)
option(VW_TEST "Turn on CMake testing target. On by default." ON)

# Add -ffast-math for speed, remove for testability.
set(linux_release_config -O3 -fomit-frame-pointer -fno-strict-aliasing -msse2 -mfpmath=sse)
set(linux_debug_config -g -O0)
set(linux_flags $<$<CONFIG:DEBUG>:${linux_debug_config}> $<$<CONFIG:RELEASE>:${linux_release_config}>)

# for profiling -- note that it needs to be gcc
if(PROFILE)
set(linux_flags ${linux_flags} -fno-strict-aliasing -pg)
endif()

# for valgrind profiling: run 'valgrind --tool=callgrind PROGRAM' then 'callgrind_annotate --tree=both --inclusive=yes'
if(VALGRIND_PROFILE)
set(linux_flags ${linux_flags} -g -fomit-frame-pointer -fno-strict-aliasing)
endif()

# gcov configuration
if(GCOV)
set(linux_flags ${linux_flags} -g -O0 -fprofile-arcs -ftest-coverage -fno-strict-aliasing -pg)
endif()

set(windows_release_config /GL /O2 /Gy /Oi /Ob2 /Ot /Oy /GT /MD)
set(windows_debug_config /INCREMENTAL /Od /Zi /analyze-)
set(windows_flags $<$<CONFIG:DEBUG>:${windows_debug_config}> $<$<CONFIG:RELEASE>:${windows_release_config}>)

# TODO move to consuming vcpkg of this dependency
add_library(rapidjson INTERFACE)
target_include_directories(rapidjson INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/rapidjson/include)

# Use folders in VS solution
set_property(GLOBAL PROPERTY USE_FOLDERS ON)

set(explore_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/explore/")

if(STATIC_LINK_VW)
set(Boost_USE_STATIC_LIBS ON)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
SET(BUILD_SHARED_LIBS OFF)
else()
set(Boost_USE_STATIC_LIBS OFF)
endif()

set(Boost_USE_MULTITHREADED ON)
set(Boost_USE_STATIC_RUNTIME OFF)

set(THREADS_PREFER_PTHREAD_FLAG ON)

# Align and foreach are also required, for some reason they can't be specified as components though.
find_package(Boost REQUIRED COMPONENTS program_options system thread unit_test_framework)
find_package(Threads REQUIRED)
find_package(ZLIB REQUIRED)

add_subdirectory(cluster)
add_subdirectory(library)
add_subdirectory(doc)
add_subdirectory(vowpalwabbit)
add_subdirectory(java)
add_subdirectory(python)

if(VW_TEST)
enable_testing()
endif()
add_subdirectory(test)

# Don't offer these make dependent targets on Windows
if(NOT WIN32)
# make bigtests BIG_TEST_ARGS="<args here>"
add_custom_target(bigtests
DEPENDS vw
COMMAND make \${BIG_TEST_ARGS}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/big_tests)
endif()

# TODO convert cs directory to cmake
# TODO convert c_test directory to cmake

# PkgConfig
configure_file(libvw.pc.in libvw.pc)
configure_file(libvw_c_wrapper.pc.in libvw_c_wrapper.pc)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libvw.pc ${CMAKE_CURRENT_BINARY_DIR}/libvw_c_wrapper.pc DESTINATION lib/pkgconfig)
254 Makefile
@@ -1,195 +1,85 @@
CXX ?= $(shell which g++)
# -- if you want to test 32-bit use this instead,
# it sometimes reveals type portability issues
# CXX = $(shell which g++) -m32
ifneq ($(CXX),)
#$(warning Using clang: "$(CXX)")
ARCH = -D__extern_always_inline=inline
else
CXX = clang++
$(warning Using clang++)
endif
#ARCH = $(shell test `$CXX -v 2>&1 | tail -1 | cut -d ' ' -f 3 | cut -d '.' -f 1,2` \< 4.3 && echo -march=nocona || echo -march=native)

ifeq ($(CXX),)
$(warning No compiler found)
exit 1
endif

UNAME := $(shell uname)
ARCH_UNAME := $(shell uname -m)
LIBS = -l boost_program_options -l pthread -l z
BOOST_INCLUDE = -I /usr/local/include/boost -I /usr/include
BOOST_LIBRARY = -L /usr/local/lib -L /usr/lib
NPROCS := 1

ifeq ($(UNAME), Linux)
BOOST_LIBRARY += -L /usr/lib/x86_64-linux-gnu
NPROCS:=$(shell grep -c ^processor /proc/cpuinfo)
endif
ifeq ($(UNAME), FreeBSD)
LIBS = -l boost_program_options -l pthread -l z -l compat
BOOST_INCLUDE = -I /usr/local/include
NPROCS:=$(shell grep -c ^processor /proc/cpuinfo)
endif
ifeq "CYGWIN" "$(findstring CYGWIN,$(UNAME))"
LIBS = -l boost_program_options-mt -l pthread -l z
BOOST_INCLUDE = -I /usr/include
NPROCS:=$(shell grep -c ^processor /proc/cpuinfo)
endif
ifeq ($(UNAME), Darwin)
LIBS = -lboost_program_options-mt -lboost_serialization-mt -l pthread -l z
# On Macs, the location isn't always clear
# brew uses /usr/local
# but /opt/local seems to be preferred by some users
# so we try them both
ifneq (,$(wildcard /usr/local/include))
BOOST_INCLUDE = -I /usr/local/include
BOOST_LIBRARY = -L /usr/local/lib
endif
ifneq (,$(wildcard /opt/local/include))
BOOST_INCLUDE = -I /opt/local/include
BOOST_LIBRARY = -L /opt/local/lib
endif
NPROCS:=$(shell sysctl -n hw.ncpu)
endif

ifneq ($(USER_BOOST_INCLUDE),)
BOOST_INCLUDE = $(USER_BOOST_INCLUDE)
endif
ifneq ($(USER_BOOST_LIBRARY),)
BOOST_LIBRARY = $(USER_BOOST_LIBRARY)
endif


JSON_INCLUDE = -I ../rapidjson/include

#LIBS = -l boost_program_options-gcc34 -l pthread -l z

ifeq ($(ARCH_UNAME), ppc64le)
OPTIM_FLAGS ?= -DNDEBUG -O3 -fomit-frame-pointer -fno-strict-aliasing #-msse2 is not supported on power
else
OPTIM_FLAGS ?= -DNDEBUG -O3 -fomit-frame-pointer -fno-strict-aliasing -msse2 -mfpmath=sse #-ffast-math #uncomment for speed, comment for testability
endif

ifeq ($(UNAME), FreeBSD)
WARN_FLAGS = -Wall
else
WARN_FLAGS = -Wall -pedantic
endif

# for normal fast execution.
FLAGS = -std=c++11 $(CFLAGS) $(LDFLAGS) $(ARCH) $(WARN_FLAGS) $(OPTIM_FLAGS) -D_FILE_OFFSET_BITS=64 $(BOOST_INCLUDE) $(JSON_INCLUDE) -fPIC #-DVW_LDA_NO_SSE

# for profiling -- note that it needs to be gcc
#FLAGS = -std=c++11 $(CFLAGS) $(LDFLAGS) $(ARCH) $(WARN_FLAGS) -O2 -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 $(BOOST_INCLUDE) $(JSON_INCLUDE) -pg -fPIC
#CXX = g++

# for valgrind / gdb debugging
#FLAGS = -std=c++11 $(CFLAGS) $(LDFLAGS) $(ARCH) $(WARN_FLAGS) -D_FILE_OFFSET_BITS=64 $(BOOST_INCLUDE) $(JSON_INCLUDE) -g -O0 -fPIC

# for valgrind profiling: run 'valgrind --tool=callgrind PROGRAM' then 'callgrind_annotate --tree=both --inclusive=yes'
#FLAGS = -std=c++11 $(CFLAGS) $(LDFLAGS) -Wall $(ARCH) -ffast-math -D_FILE_OFFSET_BITS=64 $(BOOST_INCLUDE) $(JSON_INCLUDE) -g -fomit-frame-pointer -ffast-math -fno-strict-aliasing -fPIC

FLAGS += -I ../rapidjson/include -I ../explore
BINARIES = vw active_interactor
MANPAGES = vw.1

default: vw

all: vw library_example java spanning_tree rl_clientlib

%.1: %
help2man --no-info --name="Vowpal Wabbit -- fast online learning tool" ./$< > $@

export

rl_clientlib: vw
cd reinforcement_learning/rlclientlib; $(MAKE) -j $(NPROCS) things

# Devirtualization is an optimization that changes the vtable if the compiler decides a function
# doesn't need to be virtual. This is incompatible with the mocking framework used in testing as it
# makes the vtable structure unpredictable
rl_clientlib_test: FLAGS += -fno-devirtualize
rl_clientlib_test: vw rl_clientlib
cd reinforcement_learning/unit_test; $(MAKE) -j $(NPROCS) things
(cd reinforcement_learning/unit_test && ./rlclient-test.out)

rl_example: vw rl_clientlib
cd reinforcement_learning/examples/basic_usage_cpp; $(MAKE) -j $(NPROCS) things
cd reinforcement_learning/examples/rl_sim_cpp; $(MAKE) -j $(NPROCS) things
cd reinforcement_learning/examples/test_cpp; $(MAKE) -j $(NPROCS) things
cd reinforcement_learning/examples/override_interface; $(MAKE) -j $(NPROCS) things

rl_python: vw rl_clientlib
cd reinforcement_learning/bindings/python; $(MAKE) -j $(NPROCS) things

spanning_tree:
cd cluster; $(MAKE)

vw:
cd vowpalwabbit; $(MAKE) -j $(NPROCS) things

#Target-specific flags for a profiling build. (Copied from line 70)
vw_gcov: FLAGS = -std=c++11 $(CFLAGS) $(LDFLAGS) $(ARCH) $(WARN_FLAGS) -g -O0 -fprofile-arcs -ftest-coverage -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 $(BOOST_INCLUDE) $(JSON_INCLUDE) -I ../explore -pg -fPIC #-DVW_LDA_NO_S
vw_gcov: CXX = g++
vw_gcov:
cd vowpalwabbit && env LDFLAGS="-fprofile-arcs -ftest-coverage -lgcov"; $(MAKE) -j $(NPROCS) things

active_interactor:
cd vowpalwabbit; $(MAKE)
default: vw
all: vw library_example java spanning_tree python

library_example: vw
cd library; $(MAKE) -j $(NPROCS) things
# CMake configs
ensure_cmake:
mkdir -p build
cd build; cmake ..

#Target-specific flags for a profiling build. (Copied from line 70)
library_example_gcov: FLAGS = -std=c++11 $(CFLAGS) $(LDFLAGS) $(ARCH) $(WARN_FLAGS) -g -O0 -fprofile-arcs -ftest-coverage -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 $(BOOST_INCLUDE) $(JSON_INCLUDE) -I ../explore -pg -fPIC #-DVW_LDA_NO_S
library_example_gcov: CXX = g++
library_example_gcov: vw_gcov
cd library && env LDFLAGS="-fprofile-arcs -ftest-coverage -lgcov"; $(MAKE) things
ensure_cmake_gcov:
mkdir -p build
cd build; cmake .. -DGCOV=On

python: vw
cd python; $(MAKE) things
ensure_cmake_profile:
mkdir -p build
cd build; cmake .. -DPROFILE=On

java: vw
cd java; $(MAKE) things
ensure_cmake_valgrind:
mkdir -p build
cd build; cmake .. -DVALGRIND_PROFILE=On

.FORCE:
ensure_cmake_static:
mkdir -p build
cd build; cmake .. -DSTATIC_LINK_VW=On

test: .FORCE vw library_example unit_test
@echo "vw running test-suite..."
(cd test && ./RunTests -d -fe -E 0.001 -O --onethread ../vowpalwabbit/vw)
(cd test && ./RunTests -d -fe -E 0.001 ../vowpalwabbit/vw)
cd test && python save_resume_test.py --verbose_on_fail
# Build targets
spanning_tree_build:
cd build; make -j$(shell cat ./build/nprocs.txt) spanning_tree

unit_test: vw
cd test/unit_test; $(MAKE) -j $(NPROCS) things
(cd test/unit_test && ./vw-unit-test.out)
vw_build:
cd build; make -j$(shell cat ./build/nprocs.txt) vw-bin

test_gcov: .FORCE vw_gcov library_example_gcov
@echo "vw running test-suite..."
(cd test && ./RunTests -d -fe -E 0.001 ../vowpalwabbit/vw ../vowpalwabbit/vw)
active_interactor_build:
cd build; make -j$(shell cat ./build/nprocs.txt) active_interactor

bigtests: .FORCE vw
(cd big_tests && $(MAKE) $(MAKEFLAGS))
library_example_build:
cd build; make -j$(shell cat ./build/nprocs.txt) ezexample_predict ezexample_predict_threaded ezexample_train library_example test_search search_generate recommend gd_mf_weights

install: $(BINARIES)
cd vowpalwabbit; cp $(BINARIES) /usr/local/bin; cd ../cluster; $(MAKE) install; cd ../java; $(MAKE) install;
python_build:
cd build; make -j$(shell cat ./build/nprocs.txt) pylibvw

doc:
(cd doc && doxygen Doxyfile)
java_build:
cd build; make -j$(shell cat ./build/nprocs.txt) vw_jni

test_build:
@echo "vw running test-suite..."
cd build; make -j$(shell cat ./build/nprocs.txt) test_with_output

unit_test_build:
cd build/test/unit_test; make -j$(shell cat ./build/nprocs.txt) vw-unit-test.out test

bigtests_build:
cd build; make -j$(shell cat ./build/nprocs.txt) bigtests BIG_TEST_ARGS="$(MAKEFLAGS)"

install_build:
cd build; make -j$(shell cat ./build/nprocs.txt) install

doc_build:
cd build; make doc

# These can be invoked with [gcov, valgrind, profile, static]
spanning_tree_%: ensure_cmake_% spanning_tree_build ;
vw_%: ensure_cmake_% vw_build ;
active_interactor_%: ensure_cmake_% active_interactor_build ;
library_example_%: ensure_cmake_% library_example_build ;
python_%: ensure_cmake_% python_build ;
java_%: ensure_cmake_% java_build ;
test_%: ensure_cmake_% test_build ;
unit_test_%: ensure_cmake_% unit_test_build ;
bigtests_%: ensure_cmake_% bigtests_build ;
install_%: ensure_cmake_% install_build ;

# Normal build commands that use default configuration
spanning_tree: ensure_cmake spanning_tree_build ;
vw: ensure_cmake vw_build ;
active_interactor: ensure_cmake active_interactor_build ;
library_example: ensure_cmake library_example_build ;
python: ensure_cmake python_build ;
java: ensure_cmake java_build ;
test: ensure_cmake test_build ;
unit_test: ensure_cmake unit_test_build ;
bigtests: ensure_cmake bigtests_build ;
install: ensure_cmake install_build ;

clean:
cd vowpalwabbit && $(MAKE) clean
cd cluster && $(MAKE) clean
cd library && $(MAKE) clean
cd python && $(MAKE) clean
cd java && $(MAKE) clean
cd reinforcement_learning/rlclientlib && $(MAKE) clean
cd reinforcement_learning/unit_test; $(MAKE) clean
cd reinforcement_learning/bindings/python; $(MAKE) clean
cd reinforcement_learning/examples/basic_usage_cpp; $(MAKE) clean
cd reinforcement_learning/examples/rl_sim_cpp; $(MAKE) clean
cd reinforcement_learning/examples/test_cpp; $(MAKE) clean

.PHONY: all clean install doc
rm -rf build
Oops, something went wrong.

0 comments on commit 2a8f497

Please sign in to comment.