-
Notifications
You must be signed in to change notification settings - Fork 5.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
WIP: CMake-based build system. #3100
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -148,3 +148,5 @@ GSYMS | |
/tools/cub-1.8.0.zip | ||
/tools/cub-1.8.0/ | ||
/tools/cub | ||
|
||
/build* |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
cmake_minimum_required(VERSION 3.7.2) | ||
|
||
project(Kaldi VERSION 6.0.0 LANGUAGES CXX C) | ||
|
||
include(CheckLanguage) | ||
check_language(CUDA) | ||
|
||
set(CMAKE_CXX_STANDARD 14) | ||
set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/tools/cmake" ${CMAKE_MODULE_PATH}) | ||
|
||
enable_testing() | ||
|
||
# You probably want one of "OpenBLAS", "Apple", "ATLAS", or "Intel10_64lp_seq" | ||
# Details: https://github.com/Kitware/CMake/blob/283330976f2379eec214bfe1d51a27c29cc1f8fc/Modules/FindLAPACK.cmake#L25 | ||
set(ENV{OpenBLAS_HOME} "/export/b02/ws15dgalvez/kaldi-workspace/kaldi/tools/OpenBLAS/install") | ||
# set(BLA_VENDOR "OpenBLAS" CACHE STRING "") | ||
#find_package(BLAS REQUIRED) | ||
#find_package(LAPACK REQUIRED) | ||
find_package(OpenBLAS REQUIRED) | ||
add_library(OpenBLAS SHARED IMPORTED) | ||
set_property(TARGET OpenBLAS PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${OpenBLAS_INCLUDE_DIR}) | ||
set_property(TARGET OpenBLAS PROPERTY IMPORTED_LOCATION ${OpenBLAS_LIB}) | ||
set_property(TARGET OpenBLAS PROPERTY INTERFACE_COMPILE_DEFINITIONS HAVE_OPENBLAS=1) | ||
|
||
add_library(OpenFST SHARED IMPORTED) | ||
set_property(TARGET OpenFST PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PROJECT_SOURCE_DIR}/tools/openfst/include) | ||
set_property(TARGET OpenFST PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/tools/openfst/lib/libfst.so) | ||
file(GLOB OpenFST_LIBRARIES ${PROJECT_SOURCE_DIR}/tools/openfst/lib/lib*.so ${PROJECT_SOURCE_DIR}/tools/openfst/lib/fst/lib*.so) | ||
set_property(TARGET OpenFST PROPERTY INTERFACE_LINK_LIBRARIES ${OpenFST_LIBRARIES}) | ||
|
||
find_package(Threads REQUIRED) | ||
# include("${PROJECT_SOURCE_DIR}/tools/external.cmake") | ||
include_directories("${PROJECT_SOURCE_DIR}/src") | ||
|
||
add_subdirectory(src) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
function(create_library library_name source_files test_files dependencies) | ||
add_library(${library_name} ${source_files}) | ||
target_link_libraries(${library_name} PUBLIC ${dependencies}) | ||
|
||
foreach(test ${test_files}) | ||
add_executable(${test} "${test}.cc") | ||
target_link_libraries(${test} PRIVATE ${library_name}) | ||
add_test(NAME ${library_name}-${test} COMMAND ${test}) | ||
endforeach(test) | ||
endfunction(create_library) | ||
|
||
function(create_binaries library_name source_files test_files dependencies) | ||
add_library(${library_name} ${source_files}) | ||
endfunction(create_binaries) | ||
|
||
|
||
function(create_cuda_library library_name source_files) | ||
|
||
endfunction(create_cuda_library) | ||
|
||
|
||
foreach(project base matrix util feat tree gmm transform fstext hmm lm | ||
decoder lat ) #kws) | ||
add_subdirectory(${project}) | ||
endforeach(project) | ||
|
||
# How to make a shared object which includes all of the .a's contents? | ||
# add_library(kaldi SHARED) | ||
# target_link_libraries(kaldi base) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
add_library(base kaldi-math.cc kaldi-error.cc io-funcs.cc kaldi-utils.cc timer.cc) | ||
# Used only for version.h | ||
target_include_directories(base PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/..") | ||
target_link_libraries(base PUBLIC OpenFST) | ||
add_dependencies(base generate_version_h) | ||
|
||
add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/version.h" | ||
COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/get_version.sh" | ||
COMMAND ${CMAKE_COMMAND} -E rename "${CMAKE_CURRENT_SOURCE_DIR}/version.h" "${CMAKE_CURRENT_BINARY_DIR}/version.h") | ||
add_custom_target(generate_version_h ALL | ||
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/version.h") | ||
|
||
|
||
foreach(test kaldi-math-test io-funcs-test kaldi-error-test timer-test) | ||
add_executable(${test} "${test}.cc") | ||
target_link_libraries(${test} PRIVATE base) | ||
add_test(${test} ${test}) | ||
endforeach(test) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
set(source_files training-graph-compiler.cc lattice-simple-decoder.cc | ||
lattice-faster-decoder.cc lattice-faster-online-decoder.cc simple-decoder.cc | ||
faster-decoder.cc decoder-wrappers.cc grammar-fst.cc decodable-matrix.cc) | ||
set(test_files "") | ||
set(dependencies base util matrix gmm hmm tree transform lat) | ||
create_library(decoder "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
set(source_files feature-functions.cc feature-mfcc.cc feature-fbank.cc | ||
mel-computations.cc wave-reader.cc | ||
pitch-functions.cc resample.cc online-feature.cc signal.cc | ||
feature-window.cc) | ||
|
||
set(test_files feature-mfcc-test feature-fbank-test | ||
feature-functions-test pitch-functions-test feature-sdc-test | ||
resample-test online-feature-test signal-test wave-reader-test) | ||
|
||
set(dependencies base matrix util gmm transform tree) | ||
|
||
create_library(feat "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
set(source_files push-special.cc kaldi-fst-io.cc context-fst.cc | ||
grammar-context-fst.cc) | ||
|
||
set(test_files determinize-star-test | ||
pre-determinize-test trivial-factor-weight-test | ||
context-fst-test factor-test table-matcher-test fstext-utils-test | ||
remove-eps-local-test lattice-weight-test | ||
determinize-lattice-test lattice-utils-test deterministic-fst-test | ||
push-special-test epsilon-property-test prune-special-test) | ||
|
||
set(dependencies base util matrix tree) | ||
|
||
create_library(fstext "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
set(source_files diag-gmm.cc diag-gmm-normal.cc mle-diag-gmm.cc am-diag-gmm.cc | ||
mle-am-diag-gmm.cc full-gmm.cc full-gmm-normal.cc mle-full-gmm.cc | ||
model-common.cc decodable-am-diag-gmm.cc model-test-common.cc | ||
ebw-diag-gmm.cc indirect-diff-diag-gmm.cc) | ||
|
||
set(test_files diag-gmm-test mle-diag-gmm-test full-gmm-test mle-full-gmm-test | ||
am-diag-gmm-test mle-am-diag-gmm-test ebw-diag-gmm-test) | ||
|
||
set(dependencies base util matrix tree) | ||
|
||
create_library(gmm "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
set(source_files topology.cc transitions.cc hmm-utils.cc tree-accu.cc posterior.cc | ||
hmm-test-utils.cc) | ||
set(test_files hmm-utils-test transitions-test posterior-test) | ||
set(dependencies base tree util matrix OpenFST) | ||
create_library(hmm "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
set(source_files kaldi-lattice.cc lattice-functions.cc word-align-lattice.cc | ||
phone-align-lattice.cc word-align-lattice-lexicon.cc sausages.cc | ||
push-lattice.cc minimize-lattice.cc determinize-lattice-pruned.cc | ||
confidence.cc compose-lattice-pruned.cc) | ||
|
||
set(test_files kaldi-lattice-test push-lattice-test minimize-lattice-test | ||
determinize-lattice-pruned-test word-align-lattice-lexicon-test) | ||
set(dependencies base util hmm tree matrix) | ||
create_library(lat "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
set(source_files arpa-file-parser.cc arpa-lm-compiler.cc const-arpa-lm.cc | ||
kaldi-rnnlm.cc mikolov-rnnlm-lib.cc) | ||
set(test_files arpa-file-parser-test arpa-lm-compiler-test) | ||
set(dependencies base util matrix fstext) | ||
create_library(lm "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
add_library(matrix compressed-matrix.cc kaldi-matrix.cc kaldi-vector.cc | ||
matrix-functions.cc | ||
optimization.cc packed-matrix.cc qr.cc sp-matrix.cc | ||
sparse-matrix.cc srfft.cc tp-matrix.cc) | ||
target_link_libraries(matrix PUBLIC base PRIVATE OpenBLAS) | ||
|
||
foreach(test matrix-lib-speed-test matrix-lib-test sparse-matrix-test) | ||
add_executable(${test} "${test}.cc") | ||
target_link_libraries(${test} PRIVATE matrix) | ||
add_test(${test} ${test}) | ||
endforeach(test) | ||
|
||
target_link_libraries(matrix-lib-test PRIVATE OpenBLAS) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
set(source_files lda-estimate.cc | ||
cmvn.cc transform-common.cc fmllr-diag-gmm.cc | ||
lvtln.cc mllt.cc basis-fmllr-diag-gmm.cc | ||
compressed-transform-stats.cc) | ||
|
||
set(test_files lda-estimate-test fmllr-diag-gmm-test) | ||
|
||
set(dependencies base util matrix gmm tree) | ||
|
||
create_library(transform "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
set(source_files build-tree.cc build-tree-utils.cc cluster-utils.cc | ||
tree-renderer.cc build-tree-questions.cc event-map.cc | ||
clusterable-classes.cc context-dep.cc) | ||
set(test_files context-dep-test build-tree-utils-test build-tree-test | ||
event-map-test) | ||
set(dependencies base util matrix) | ||
create_library(tree "${source_files}" "${test_files}" "${dependencies}") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
add_library(util kaldi-holder.cc kaldi-table.cc kaldi-thread-test.cc | ||
kaldi-thread.cc kaldi-io.cc kaldi-semaphore.cc parse-options.cc | ||
simple-io-funcs.cc simple-options.cc text-utils.cc) | ||
target_link_libraries(util PUBLIC base PUBLIC matrix PRIVATE Threads::Threads) | ||
foreach(test const-integer-set-test edit-distance-test hash-list-test kaldi-io-test parse-options-test simple-options-test stl-utils-test text-utils-test kaldi-table-test) | ||
add_executable(${test} "${test}.cc") | ||
target_link_libraries(${test} PRIVATE util) | ||
add_test(${test} ${test}) | ||
endforeach(test) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
SET(Open_BLAS_INCLUDE_SEARCH_PATHS | ||
/usr/include | ||
/usr/include/openblas | ||
/usr/include/openblas-base | ||
/usr/local/include | ||
/usr/local/include/openblas | ||
/usr/local/include/openblas-base | ||
/opt/OpenBLAS/include | ||
$ENV{OpenBLAS_HOME} | ||
$ENV{OpenBLAS_HOME}/include | ||
) | ||
|
||
SET(Open_BLAS_LIB_SEARCH_PATHS | ||
$ENV{OpenBLAS}cd | ||
$ENV{OpenBLAS}/lib | ||
$ENV{OpenBLAS_HOME} | ||
$ENV{OpenBLAS_HOME}/lib | ||
/lib/ | ||
/lib/openblas-base | ||
/lib64/ | ||
/usr/lib | ||
/usr/lib/openblas-base | ||
/usr/lib64 | ||
/usr/local/lib | ||
/usr/local/lib64 | ||
/opt/OpenBLAS/lib | ||
) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Question: does CMake support searching for includes/libs the way autoconf does, i. e. compiling or linking one-liners to see if they succeed? Basically, this: #3228. We can extend this list as infinitum (I ended up with the OpenBLAS package in the path like /opt/rh/epel/OpenBLAS on CentOS, IIR the path correctly), but the toolset is usually the best guy to let figure out it's own libraries. If linking with Also, regarding OpenBLAS only, I am hacking configure now in an assumption that the best OpenBLAS is in our tools/OpenBLAS/install/{include,lib} provided it exists, as it is the one the most optimized for the user's system. It also drops some files into the cmake/ subdirectory of one of these (I do not remember which, sorry, I'll check), apparently to help CMake builds find it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FYI, if you find it helpful, here's the contents of the two files in that
What I do not see here is a pointer to the .a version of the library. The build builds both .so and .a, but these files which (I assume) help cmake discover the package do not mention the available static lib. We support
Maybe sending them a PR that would fix that (if the above files are not canonical, in cmake sense) would get it in in time you'd have a chance to continue this work, so at the least no hack like splitting the value of the variable These files are made by their standard Makefile build from the latest release 0.3.5, not the development branch. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. for a library named foo
|
||
|
||
FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS}) | ||
FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS}) | ||
|
||
SET(OpenBLAS_FOUND ON) | ||
|
||
# Check include files | ||
IF(NOT OpenBLAS_INCLUDE_DIR) | ||
SET(OpenBLAS_FOUND OFF) | ||
MESSAGE(STATUS "Could not find OpenBLAS include. Turning OpenBLAS_FOUND off") | ||
ENDIF() | ||
|
||
# Check libraries | ||
IF(NOT OpenBLAS_LIB) | ||
SET(OpenBLAS_FOUND OFF) | ||
MESSAGE(STATUS "Could not find OpenBLAS lib. Turning OpenBLAS_FOUND off") | ||
ENDIF() | ||
|
||
IF (OpenBLAS_FOUND) | ||
IF (NOT OpenBLAS_FIND_QUIETLY) | ||
MESSAGE(STATUS "Found OpenBLAS libraries: ${OpenBLAS_LIB}") | ||
MESSAGE(STATUS "Found OpenBLAS include: ${OpenBLAS_INCLUDE_DIR}") | ||
ENDIF (NOT OpenBLAS_FIND_QUIETLY) | ||
ELSE (OpenBLAS_FOUND) | ||
IF (OpenBLAS_FIND_REQUIRED) | ||
MESSAGE(FATAL_ERROR "Could not find OpenBLAS") | ||
ENDIF (OpenBLAS_FIND_REQUIRED) | ||
ENDIF (OpenBLAS_FOUND) | ||
|
||
MARK_AS_ADVANCED( | ||
OpenBLAS_INCLUDE_DIR | ||
OpenBLAS_LIB | ||
OpenBLAS | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
include(ExternalProject) | ||
|
||
# set(CUB_VERSION 1.8.0) | ||
|
||
# ExternalProject_Add(cub | ||
# URL https://github.com/NVlabs/cub/archive/${CUB_VERSION}.zip | ||
# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/cub | ||
# BUILD_IN_SOURCE 1 | ||
# CONFIGURE_COMMAND : | ||
# BUILD_COMMAND : | ||
# INSTALL_COMMAND : | ||
# ) | ||
|
||
# set(OPENFST_VERSION 1.6.7) | ||
|
||
# ExternalProject_Add(openfst | ||
# URL http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.7.tar.gz | ||
# # PREFIX ${CMAKE_CURRENT_BINARY_DIR}/openfst | ||
# BUILD_IN_SOURCE 1 | ||
# #SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/openfst | ||
# INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/openfst-install | ||
# CONFIGURE_COMMAND ./configure --prefix=${CMAKE_CURRENT_BINARY_DIR}/openfst-install --enable-static --enable-shared --enable-far --enable-ngram-fsts | ||
# BUILD_COMMAND make -j 8 | ||
# INSTALL_COMMAND make install | ||
# ) | ||
set(OPENFST_BIN_DIRS ${CMAKE_CURRENT_BINARY_DIR}/openfst-install/bin) | ||
set(OPENFST_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/openfst-install/include) | ||
# TODO(galv): Figure out how Find*.cmake files choose between .so and .a files. | ||
file(GLOB OPENFST_LIBRARIES ${openfst_PREFIX}/lib/lib*.so ${openfst_PREFIX}/lib/fst/lib*.so) | ||
|
||
|
||
# find_package(OpenBLAS REQUIRED) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
i assume this is temporary?