Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use CMakeLists.txt instead of Makefile #4

Merged
merged 41 commits into from
Jun 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
65e5b0b
Use CMakeLists.txt instead of Makefile
adamant-pwn Apr 24, 2024
106eb03
Update CMakeLists.txt
adamant-pwn May 13, 2024
660268d
Refactor CMakeLists.txt
adamant-pwn May 15, 2024
3fbcda9
Update
adamant-pwn May 15, 2024
0b10e4e
Use .a instead of .so, link pod5 with zstd
adamant-pwn May 15, 2024
d0863fc
Fix docker failure
adamant-pwn May 15, 2024
837cd9b
Update README.md
adamant-pwn May 15, 2024
6d530c7
Update README.md
adamant-pwn May 15, 2024
ae2e122
Add basic Dockerfile
adamant-pwn May 13, 2024
45d6504
Update Dockerfile
adamant-pwn May 15, 2024
1804191
Make test workflow
adamant-pwn May 15, 2024
20a273d
Checkout recursive
adamant-pwn May 15, 2024
a07df3e
Update branch name in workflow
adamant-pwn May 15, 2024
0882d3f
Redundant include <array>
adamant-pwn May 15, 2024
1617247
Use -h in GH workflow
adamant-pwn May 16, 2024
7b2c9d2
Run workflow for any PR
adamant-pwn May 16, 2024
f445400
Remove python bindings WIP, add proper handling of C/CXX
adamant-pwn May 16, 2024
35cacaf
Testing simplified RUCLIENT_ENABLED
adamant-pwn May 16, 2024
d894b6e
Use TARGET_NAME as an argument
adamant-pwn May 16, 2024
8d317cb
Try caching for workflows
adamant-pwn May 16, 2024
2a320c8
Try NOHDF5=ON by default
adamant-pwn May 16, 2024
33ce9dc
Try NOPOD5=ON
adamant-pwn May 16, 2024
722cf0f
Update
adamant-pwn May 17, 2024
ddcb274
Remove readuntil_fake
adamant-pwn May 17, 2024
721bee5
Remove log.txt
adamant-pwn May 17, 2024
001425e
Try build with POD5 and HDF5
adamant-pwn May 17, 2024
8f273f4
merged tflite branch and fixed cmake setup
maximilianmordig May 24, 2024
f8f053c
adding some fixes to the cmake file, and prototyping python integration
maximilianmordig May 24, 2024
dd0ac71
Fix slow5lib.so
adamant-pwn May 31, 2024
95fd6ae
wrote crude wrapper around rawhash for signal mapping
maximilianmordig Jun 3, 2024
1351418
added option to skip querying revcomp signal, small fixes
maximilianmordig Jun 4, 2024
e97d2fd
confirmed that alignment is the same for a single read
maximilianmordig Jun 4, 2024
4d6dbf5
cleaned up rawhash python wrapper
maximilianmordig Jun 4, 2024
a70f641
Try caching build directory
adamant-pwn Jun 5, 2024
1afd7a1
Lowercase repo owner
adamant-pwn Jun 5, 2024
feba398
Use CACHE STRING on ccache
adamant-pwn Jun 5, 2024
1f439d8
Remove build from .dockerignore for caching
adamant-pwn Jun 5, 2024
619ccce
Only run build-push-action once
adamant-pwn Jun 5, 2024
f4319f2
Try -j 2 with cached build directory
adamant-pwn Jun 5, 2024
63e1001
Use 3 cores and only cache build dir
adamant-pwn Jun 5, 2024
fcea20c
Try to use less space in workflows
adamant-pwn Jun 6, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions .github/workflows/docker-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
name: Docker Build and Test

on:
push:
branches:
- main
pull_request:

jobs:
build:
runs-on: ubuntu-latest

steps:
- name: Delete huge unnecessary tools folder
run: rm -rf /opt/hostedtoolcache

- name: Checkout repository
uses: actions/checkout@v3
with:
submodules: recursive

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2

- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Cache build directory
uses: actions/cache@v3
with:
path: build
key: ${{ runner.os }}-build-${{ github.sha }}
restore-keys: |
${{ runner.os }}-build-

- name: Set lowercase repository owner
id: repo_owner
run: echo "::set-output name=owner::${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]'

- name: Build Docker image
uses: docker/build-push-action@v4
with:
context: .
load: true
tags: ghcr.io/${{ steps.repo_owner.outputs.owner }}/build_test:latest
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
BUILDKIT_INLINE_CACHE=1

- name: Extract build directory from Docker image
run: |
container_id=$(docker create ghcr.io/${{ steps.repo_owner.outputs.owner }}/build_test:latest)
docker cp $container_id:/rawhash2/build ./build
docker rm $container_id

- name: Run Docker container
run: docker run --rm ghcr.io/${{ steps.repo_owner.outputs.owner }}/build_test:latest -h
9 changes: 7 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
bin/
build/
run_dir/
.venv/
example_out/
__pycache__/

test/data/d1_*
test/data/d2_*
Expand Down Expand Up @@ -72,7 +77,7 @@ test/eval/*.summary
*/.DS_Store
test/*.time
test/*.idx
src/*.o
#src/*.o

extern/pod5*

Expand All @@ -86,4 +91,4 @@ test/evaluation/read_mapping/*parameters/
test/evaluation/read_mapping/s_modify.py

extern/tensorflow/tflite_build/
test/evaluation/read_mapping/*.features
test/evaluation/read_mapping/*.features
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
cmake_minimum_required(VERSION 3.10)
project(RawHash2)

add_subdirectory(src)
16 changes: 16 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
FROM gcc:latest

RUN apt-get update && apt-get install -y \
cmake make mold ccache \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /rawhash2
COPY . /rawhash2

RUN mkdir -p build && cd build \
&& cmake .. \
&& make -j 3

ENTRYPOINT ["./build/bin/rawhash2"]

LABEL Name=rawhash2 Version=0.0.1
29 changes: 0 additions & 29 deletions Makefile

This file was deleted.

44 changes: 27 additions & 17 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,46 +32,56 @@ RawHash performs real-time mapping of nanopore raw signals. When the prefix of r

# Installation

* Clone the code from its GitHub repository (`--recursive` must be used):
* Clone the code from its GitHub repository and recursively initialize submodules:

```bash
git clone --recursive https://github.com/CMU-SAFARI/RawHash.git rawhash2
git clone https://github.com/CMU-SAFARI/RawHash.git rawhash2
cd rawhash2 && git submodule update --init --recursive
```

* Compile (Make sure you have a C++ compiler and GNU make):

```bash
cd rawhash2 && make
# if not doing a fresh clone, make sure that the submodules don't have anything built from previous makefile-based
# setup , i.e. delete extern directory, then initialize submodules as above
(mkdir -p build && cd build && cmake .. && make -j)
build/bin/rawhash2
```

If the compilation is successful, the path to the binary will be `bin/rawhash2`.
Troubleshooting:
- `makefile error 2`: rerun `make -j`, then the actual error is shown
- updating submodules: the current cmake setup may not correctly handle this, so the easiest solution is to delete the build directory

If the compilation is successful, the default path to the binary will be `build/bin/rawhash2`.

## Compiling with HDF5, SLOW5, and POD5

We are aware that some of the pre-compiled libraries (e.g., POD5) may not work in your system and you may need to compile these libraries from scratch. Additionally, it may be possible that you may not want to compile any of the HDF5, SLOW5, or POD5 libraries if you are not going to use them. RawHash2 provides a flexible Makefile to enable custom compilation of these libraries.
We are aware that some of the pre-compiled libraries (e.g., POD5) may not work in your system and you may need to compile these libraries from scratch. Additionally, it may be possible that you may not want to compile any of the HDF5, SLOW5, or POD5 libraries if you are not going to use them. RawHash2 provides several CMake options to enable custom compilation of these libraries.

* It is possible to provide your own include and lib directories for *any* of the HDF5, SLOW5, and POD5 libraries, if you do not want to use the source code or the pre-compiled binaries that come with RawHash2. To use your own include and lib directories you should pass them to `make` when compiling as follows:
It is possible to provide your own include and lib directories for *any* of the HDF5, SLOW5, and POD5 libraries, if you do not want to use the source code or the pre-compiled binaries that come with RawHash2. To use your own include and lib directories you should pass them to `cmake` when compiling as follows:

```bash
#Provide the path to all of the HDF5/SLOW5/POD5 include and lib directories during compilation
make HDF5_INCLUDE_DIR=/path/to/hdf5/include HDF5_LIB_DIR=/path/to/hdf5/lib \
SLOW5_INCLUDE_DIR=/path/to/slow5/include SLOW5_LIB_DIR=/path/to/slow5/lib \
POD5_INCLUDE_DIR=/path/to/pod5/include POD5_LIB_DIR=/path/to/pod5/lib
# Provide the path to all of the HDF5/SLOW5/POD5 include and lib directories during compilation
cmake -DHDF5_DIR=/path/to/hdf5 -DSLOW5_DIR=/path/to/slow5 -DPOD5_DIR=/path/to/pod5 ..

#Provide the path to only POD5 include and lib directories during compilation
make POD5_INCLUDE_DIR=/path/to/pod5/include POD5_LIB_DIR=/path/to/pod5/lib
# Provide the path to only POD5 include and lib directories during compilation
cmake -DPOD5_DIR=/path/to/pod5
```

* It is possible to disable compiling *any* of the HDF5, SLOW5, and POD5 libraries. To disable them, you can use the following variables
Note that the provided path should generally contain _both_ `include/` and `lib/` folders with the corresponding project's include and library files.

It is possible to disable compiling *any* of the HDF5, SLOW5, and POD5 libraries. To disable them, you can use the following variables

```bash
#Disables compiling HDF5
make NOHDF5=1
# Disables compiling HDF5
cmake -DNOHDF5=1 ..

#Disables compiling SLOW5 and POD5
make NOSLOW5=1 NOPOD5=1
# Disables compiling SLOW5 and POD5
cmake -DNOSLOW5=1 -DNOPOD5=1 ..
```

The variables and paths will be stored in CMake cache, meaning that you would need to run `cmake` again with explicitly provided new values to change them.

# Usage

## Getting help
Expand Down
57 changes: 57 additions & 0 deletions cmake/SetupCCacheMold.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
function(enable_ccache)
# export PATH="/usr/lib/ccache:$PATH"
find_program(CCACHE_EXE ccache)
if(CCACHE_EXE)
message(STATUS "found ccache at ${CCACHE_EXE}, using it")
set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_EXE}" CACHE STRING "C compiler launcher")
set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXE}" CACHE STRING "C++ compiler launcher")
else()
message(STATUS "ccache not found, not using it")
endif()
endfunction()

# mold is a much faster linker than ld, also see for mold: https://github.com/heavyai/heavydb/blob/master/CMakeLists.txt
# or try: https://gitlab.kitware.com/cmake/cmake/-/merge_requests/8861, can now use CMAKE_LINKER_TYPE
macro(set_alternate_linker linker)
find_program(LINKER_EXECUTABLE ld.${linker} ${linker})
if(LINKER_EXECUTABLE)
message(STATUS "Found linker ${linker}: ${LINKER_EXECUTABLE}")
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS 12.0.0)
add_link_options("-ld-path=${linker}")
elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS 12.1.0 AND "${linker}" STREQUAL "mold")
# LINKER_EXECUTABLE will be a full path to ld.mold, so we replace the end of the path, resulting in the relative
# libexec/mold dir, and tell GCC to look there first for an override version of executables, in this case, ld
string(REPLACE "bin/ld.mold" "libexec/mold" PATH_TO_LIBEXEC_MOLD ${LINKER_EXECUTABLE})
add_link_options("-B${PATH_TO_LIBEXEC_MOLD}")
else()
add_link_options("-fuse-ld=${linker}")
endif()
else()
message(FATAL_ERROR "Could not find linker ${linker}")
endif()
endmacro()

# not working
# function(setup_ccache_mold)
# if(USE_CCACHE)
# find_program(CCACHE_PROGRAM ccache)
# if(CCACHE_PROGRAM)
# message(STATUS "ccache found: ${CCACHE_PROGRAM}")
# set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
# set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
# else()
# message(WARNING "ccache not found, using default compiler")
# endif()
# endif()

# if(USE_MOLD)
# find_program(MOLD_PROGRAM mold)
# # todo: does not seem to work, for a working configuration, see https://github.com/ratschlab/readuntil_fake/blob/refactor/cmake/utils.cmake
# if(MOLD_PROGRAM)
# message(STATUS "mold found: ${MOLD_PROGRAM}")
# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${MOLD_PROGRAM}")
# else()
# message(WARNING "mold not found, using default linker")
# endif()
# endif()
# endfunction()
32 changes: 32 additions & 0 deletions cmake/SetupHDF5.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
include(${CMAKE_CURRENT_LIST_DIR}/Util.cmake)

function(setup_hdf5 TARGET_NAME)
if(NOHDF5)
target_compile_definitions(${TARGET_NAME} PRIVATE NHDF5RH=1)
adamant-pwn marked this conversation as resolved.
Show resolved Hide resolved
else()
# print HDF5_DIR
message(STATUS "EXTERNAL_PROJECTS_BUILD_DIR: ${EXTERNAL_PROJECTS_BUILD_DIR}")
message(STATUS "HDF5_DIR: ${HDF5_DIR}")
set(HDF5_SOURCE_DIR ${CMAKE_SOURCE_DIR}/extern/hdf5)
if(HDF5_COMPILE)
if(NOT HDF5_DIR)
override_cached(HDF5_DIR ${EXTERNAL_PROJECTS_BUILD_DIR}/hdf5)
endif()
set(HDF5_BUILD_DIR ${HDF5_DIR}/build)
ExternalProject_Add(
hdf5_build
SOURCE_DIR ${HDF5_SOURCE_DIR}
BINARY_DIR ${HDF5_BUILD_DIR}
CONFIGURE_COMMAND ${HDF5_SOURCE_DIR}/configure --enable-threadsafe --disable-hl --prefix=${HDF5_BUILD_DIR}
# INSTALL_DIR and DCMAKE_INSTALL_PREFIX are ignored by hdf5
INSTALL_COMMAND make install prefix=${HDF5_DIR}
)
add_dependencies(${TARGET_NAME} hdf5_build)
else()
if(NOT HDF5_DIR)
message(FATAL_ERROR "HDF5_COMPILE is OFF, but no dir provided")
endif()
endif()
link_imported_library(${TARGET_NAME} hdf5 ${HDF5_DIR})
endif()
endfunction()
Loading
Loading