diff --git a/.github/workflows/static-binary.yaml b/.github/workflows/static-binary.yaml new file mode 100644 index 00000000000..c0619c9c0d8 --- /dev/null +++ b/.github/workflows/static-binary.yaml @@ -0,0 +1,69 @@ +name: "VAST Static" +on: + repository_dispatch: + types: static-binary + push: + branches: + - master + pull_request: + paths: + - '.github/workflows/static-binary.yaml' + release: + types: published + +jobs: + static_binary: + runs-on: ubuntu-latest + name: Static Binary + env: + BUILD_DIR: build + STATIC_BINARY_TARGET: ${{ github.event.client_payload.args == '' && 'vast' || github.event.client_payload.args }} + steps: + - name: Checkout + if: github.event_name == 'repository_dispatch' + uses: actions/checkout@v1 + with: + ref: ${{ github.event.client_payload.ref == '' && 'master' || github.event.client_payload.ref }} + + - name: Checkout + if: github.event_name != 'repository_dispatch' + uses: actions/checkout@v1 + + - name: Install Nix + uses: cachix/install-nix-action@v10 + + - name: Setup Cachix + uses: cachix/cachix-action@v6 + with: + name: vast + signingKey: '${{ secrets.CACHIX_VAST_SIGNING_KEY }}' + + - name: Build a Static Binary + run: | + nix/static-binary.sh --use-head + + - name: Create Paths + id: create_paths + run: | + ARTIFACT_NAME=$(ls "${BUILD_DIR}" | grep "vast.*.tar.gz") + echo "::set-output name=artifact_name::${ARTIFACT_NAME}" + + - name: Upload Artifact to Github + uses: actions/upload-artifact@v1 + with: + name: "${{ steps.create_paths.outputs.artifact_name }}" + path: "${{ env.BUILD_DIR }}/${{ steps.create_paths.outputs.artifact_name }}" + + - name: Publish to GitHub Release + if: github.event_name == 'release' && github.event.action == 'published' + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: "${{ env.BUILD_DIR }}/${{ steps.create_paths.outputs.artifact_name }}" + # The asset name is constant so we can permanently link to + # https://github.com/tenzir/vast/releases/latest/download/vast-linux-static.tar.gz + # for a build of the latest release. + asset_name: "vast-linux-static.tar.gz" + asset_content_type: application/gzip diff --git a/.github/workflows/vast.yaml b/.github/workflows/vast.yaml index d0dcee19a18..e4cf99b5ca7 100644 --- a/.github/workflows/vast.yaml +++ b/.github/workflows/vast.yaml @@ -119,9 +119,11 @@ jobs: id: configure_env run: | PACKAGE_NAME="$(echo "vast-$(git describe)-$(uname -s)-${{ matrix.configure.tag }}-${{ matrix.os.compiler }}" | awk '{ print tolower($0) }')" + PUBLISH_NAME="$(echo "vast-$(uname -s)-${{ matrix.configure.tag }}-${{ matrix.os.compiler }}" | awk '{ print tolower($0) }')" BUILD_DIR="build" ## The upload artifact action cannot resolve environment variables. echo "::set-output name=package_name::$PACKAGE_NAME" + echo "::set-output name=publish_name::$PUBLISH_NAME" echo "::set-output name=build_dir::$BUILD_DIR" echo "::set-env name=PACKAGE_NAME::$PACKAGE_NAME" echo "::set-env name=BUILD_DIR::$BUILD_DIR" @@ -261,5 +263,10 @@ jobs: with: upload_url: ${{ github.event.release.upload_url }} asset_path: "${{ steps.configure_env.outputs.build_dir }}/${{ steps.configure_env.outputs.package_name }}.tar.gz" - asset_name: "${{ steps.configure_env.outputs.package_name }}.tar.gz" + # The asset names are constant so we can permanently link to + # https://github.com/tenzir/vast/releases/latest/download/vast-ubuntu-release-gcc.tar.gz + # https://github.com/tenzir/vast/releases/latest/download/vast-ubuntu-release-clang.tar.gz + # https://github.com/tenzir/vast/releases/latest/download/vast-darwin-release-appleclang.tar.gz + # for builds of the latest release. + asset_name: "${{ steps.configure_env.outputs.publish_name }}.tar.gz" asset_content_type: application/gzip diff --git a/.gitignore b/.gitignore index 189de347073..5f7341ea8a5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ # Build artifacts /build*/ +/nix/build/ /compile_commands.json .mypy_cache/ __pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 92dfd71b5c1..e0cff7f524f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ Every entry has a category for which we use the following visual abbreviations: ## Unreleased +- 🎁 Starting with this release, installing VAST on any Linux becomes + significantly easier: A static binary will be provided with each release on + the GitHub releases page. [#966](https://github.com/tenzir/vast/pull/966) + - 🐞 The PCAP reader now correctly shows the amount of generated events. [#954](https://github.com/tenzir/vast/pull/954) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2fea1a1bfc8..4490cea84b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,8 +82,8 @@ option(VAST_ENABLE_ASSERTIONS "Enable Assertions" option(VAST_RELOCATABLE_INSTALL "Enable relocatable installations" ON) option(VAST_USE_BUNDLED_CAF "Always use the CAF submodule" OFF) option(ENABLE_ZEEK_TO_VAST "Build zeek-to-vast" ON) -option(VAST_STATIC_EXECUTABLE "Link VAST statically") -option(VAST_USE_JEMALLOC "Use jemalloc instead of libc malloc" OFF) +option(VAST_STATIC_EXECUTABLE "Link VAST statically" $ENV{VAST_STATIC_EXECUTABLE}) +option(VAST_USE_JEMALLOC "Use jemalloc instead of libc malloc" "${VAST_STATIC_EXECUTABLE}") cmake_dependent_option( BUILD_SHARED_LIBS "Build shared libraries instead of static" ON "NOT VAST_STATIC_EXECUTABLE" OFF) diff --git a/README.md b/README.md index c2d3229f53a..636df498fc5 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,9 @@ The network telemetry engine for data-driven security investigations. [![Build Status][ci-badge]][ci-url] +[![Static Build Status][ci-static-badge]][ci-static-url] [![Examples Status][ci-examples-badge]][ci-examples-url] -[![Docker Status][docker-badge]][ci-docker-url] +[![Docker Status][ci-docker-badge]][ci-docker-url] [![Changelog][changelog-badge]][changelog-url] [![Latest Release][latest-release-badge]][latest-release-url] [![Chat][chat-badge]][chat-url] @@ -136,10 +137,12 @@ proceedings][nsdi-proceedings]. [chat-url]: https://gitter.im/tenzir/chat [ci-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3AVAST [ci-badge]: https://github.com/tenzir/vast/workflows/VAST/badge.svg?branch=master -[ci-examples-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3AJupyter+Notebook +[ci-examples-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3A%22Jupyter+Notebook%22 [ci-examples-badge]: https://github.com/tenzir/vast/workflows/Jupyter%20Notebook/badge.svg?branch=master -[ci-docker-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3AVAST+Docker +[ci-docker-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3A%22VAST+Docker%22 [ci-docker-badge]: https://github.com/tenzir/vast/workflows/VAST%20Docker/badge.svg?branch=master +[ci-static-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3A%22VAST+Static%22 +[ci-static-badge]: https://github.com/tenzir/vast/workflows/VAST%20Static/badge.svg?branch=master [license-badge]: https://img.shields.io/badge/license-BSD-blue.svg [license-url]: https://raw.github.com/vast-io/vast/master/COPYING [changelog-badge]: https://img.shields.io/badge/view-changelog-green.svg diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake index f2a45bc06b5..98a11403afc 100644 --- a/cmake/FindZSTD.cmake +++ b/cmake/FindZSTD.cmake @@ -23,6 +23,13 @@ set(ZSTD_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}zstd${ZSTD_STATIC_LIB_SUFFIX}) # First, find via if specified ZTD_ROOT +if (NOT ZSTD_ROOT) + # We look for an environment variable of the same name too, because + # that can be provided by a tool like nix-shell or docker. Wo do this + # so the user doesn't have to remember to add "-DZSTD_ROOT=..." to their + # cmake invocations. + set(ZSTD_ROOT "$ENV{ZSTD_ROOT}") +endif () if (ZSTD_ROOT) message(STATUS "Using ZSTD_ROOT: ${ZSTD_ROOT}") find_library( diff --git a/cmake/Findjemalloc.cmake b/cmake/Findjemalloc.cmake index 1ff9d6373ed..8683123b03d 100644 --- a/cmake/Findjemalloc.cmake +++ b/cmake/Findjemalloc.cmake @@ -7,7 +7,7 @@ else () endif () if (jemalloc_ROOT_DIR) - message(STATUS "Using jemalloc_ROOT: ${jemalloc_ROOT}") + message(STATUS "Using jemalloc_ROOT: ${jemalloc_ROOT_DIR}") find_path( jemalloc_INCLUDE_DIR NAMES jemalloc/jemalloc.h diff --git a/nix/arrow/fix-static-jemalloc.patch b/nix/arrow/fix-static-jemalloc.patch new file mode 100644 index 00000000000..0ff08259b49 --- /dev/null +++ b/nix/arrow/fix-static-jemalloc.patch @@ -0,0 +1,67 @@ +diff --git a/cmake_modules/ArrowJemallocTarget.cmake.in b/cmake_modules/ArrowJemallocTarget.cmake.in +new file mode 100644 +index 000000000..123fdf4b7 +--- /dev/null ++++ b/cmake_modules/ArrowJemallocTarget.cmake.in +@@ -0,0 +1,18 @@ ++ ++# Compute the installation prefix relative to this file. ++get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) ++get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) ++get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) ++get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) ++if(_IMPORT_PREFIX STREQUAL "/") ++ set(_IMPORT_PREFIX "") ++endif() ++ ++add_library(jemalloc::jemalloc STATIC IMPORTED) ++set_target_properties(jemalloc::jemalloc ++ PROPERTIES INTERFACE_LINK_LIBRARIES ++ Threads::Threads) ++set_property(TARGET jemalloc::jemalloc APPEND PROPERTY IMPORTED_CONFIGURATIONS @CMAKE_BUILD_TYPE@) ++set_target_properties(jemalloc::jemalloc PROPERTIES ++ IMPORTED_LINK_INTERFACE_LANGUAGES_@CMAKE_BUILD_TYPE@ "C;CXX" ++ IMPORTED_LOCATION_@CMAKE_BUILD_TYPE@ "${_IMPORT_PREFIX}/lib/arrow/@JEMALLOC_STATIC_NAME@") +diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake +index ba3138d20..2722968fd 100644 +--- a/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cmake_modules/ThirdpartyToolchain.cmake +@@ -1308,8 +1308,9 @@ if(ARROW_JEMALLOC) + set(ARROW_JEMALLOC_USE_SHARED OFF) + set(JEMALLOC_PREFIX + "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/") ++ set(JEMALLOC_STATIC_NAME "libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(JEMALLOC_STATIC_LIB +- "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}") ++ "${JEMALLOC_PREFIX}/lib/${JEMALLOC_STATIC_NAME}") + set(JEMALLOC_CONFIGURE_COMMAND ./configure "AR=${CMAKE_AR}" "CC=${CMAKE_C_COMPILER}") + if(CMAKE_OSX_SYSROOT) + list(APPEND JEMALLOC_CONFIGURE_COMMAND "SDKROOT=${CMAKE_OSX_SYSROOT}") +@@ -1353,6 +1354,14 @@ if(ARROW_JEMALLOC) + INTERFACE_INCLUDE_DIRECTORIES + "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src") + add_dependencies(jemalloc::jemalloc jemalloc_ep) ++ if(ARROW_BUILD_STATIC) ++ install(FILES "${JEMALLOC_STATIC_LIB}" ++ DESTINATION "${CMAKE_INSTALL_LIBDIR}/arrow") ++ configure_file("${PROJECT_SOURCE_DIR}/cmake_modules/ArrowJemallocTarget.cmake.in" ++ "${CMAKE_CURRENT_BINARY_DIR}/ArrowJemallocTarget.cmake" @ONLY) ++ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ArrowJemallocTarget.cmake" ++ DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/arrow") ++ endif() + endif() + + # ---------------------------------------------------------------------- +diff --git a/src/arrow/ArrowConfig.cmake.in b/src/arrow/ArrowConfig.cmake.in +index 0e595066d..c52e3cb07 100644 +--- a/src/arrow/ArrowConfig.cmake.in ++++ b/src/arrow/ArrowConfig.cmake.in +@@ -39,5 +39,8 @@ include("${CMAKE_CURRENT_LIST_DIR}/ArrowOptions.cmake") + # Load targets only once. If we load targets multiple times, CMake reports + # already existent target error. + if(NOT (TARGET arrow_shared OR TARGET arrow_static)) ++ if(ARROW_BUILD_STATIC AND ARROW_JEMALLOC) ++ include("${CMAKE_CURRENT_LIST_DIR}/ArrowJemallocTarget.cmake") ++ endif() + include("${CMAKE_CURRENT_LIST_DIR}/ArrowTargets.cmake") + endif() diff --git a/nix/broker/default.nix b/nix/broker/default.nix new file mode 100644 index 00000000000..07ea980e2d7 --- /dev/null +++ b/nix/broker/default.nix @@ -0,0 +1,47 @@ +{ stdenv, lib, fetchgit, cmake, caf, openssl, python, ncurses +, static ? stdenv.hostPlatform.isMusl +, linkTimeOptimization ? static }: + +let + source = builtins.fromJSON (builtins.readFile ./source.json); + isCross = stdenv.buildPlatform != stdenv.hostPlatform; + fixODR = static && linkTimeOptimization; +in + +stdenv.mkDerivation rec { + pname = "broker"; + version = builtins.substring 0 10 source.date; + + src = lib.callPackageWith source fetchgit {}; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ caf openssl ] + ++ lib.optionals (!static) [ python ncurses ]; + + cmakeFlags = [ + "-DBUILD_SHARED_LIBS=ON" + "-DBROKER_DISABLE_DOCS=ON" + "-DCAF_ROOT_DIR=${caf}" + "-DPY_MOD_INSTALL_DIR=${placeholder "out"}/${python.sitePackages}" + ] ++ lib.optionals static [ + "-DENABLE_STATIC_ONLY=ON" + "-DOPENSSL_USE_STATIC_LIBS=TRUE" + ] ++ lib.optionals linkTimeOptimization [ + "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW" + "-DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=ON" + ] ++ lib.optionals isCross [ + "-DBROKER_DISABLE_TESTS=ON" + ]; + hardeningDisable = lib.optional static "pic"; + dontStrip = static; + + patches = [ ./fix_static_linkage.patch ]; + + meta = with lib; { + description = "Zeek networking layer"; + homepage = http://zeek.io/; + license = licenses.bsd3; + platforms = platforms.unix; + maintainers = with maintainers; [ tobim ]; + }; +} diff --git a/nix/broker/fix_static_linkage.patch b/nix/broker/fix_static_linkage.patch new file mode 100644 index 00000000000..b1bdb02a8b4 --- /dev/null +++ b/nix/broker/fix_static_linkage.patch @@ -0,0 +1,307 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 5461f8c..563250f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -63,10 +63,15 @@ endif() + + # -- Dependencies ------------------------------------------------------------- + ++# OpenSSL ++find_package(OpenSSL REQUIRED) ++set(LINK_LIBS ${LINK_LIBS} OpenSSL::SSL) ++ + if ( CAF_ROOT_DIR ) + find_package(CAF COMPONENTS openssl test io core REQUIRED) + set(caf_dir "${CAF_LIBRARY_CORE}") +- set(caf_config "${CAF_INCLUDE_DIR_CORE}/caf/config.hpp") ++ list(GET CAF_INCLUDE_DIR_CORE 0 caf_core_include) ++ find_file(caf_config "caf/config.hpp" PATHS ${caf_core_include} NO_DEFAULT_PATH) + get_filename_component(caf_dir ${caf_dir} PATH) + + if (EXISTS "${caf_dir}/../libcaf_core") +@@ -313,11 +318,7 @@ if (CAF_VERSION VERSION_LESS ${CAF_VERSION_REQUIRED}) + endif () + + include_directories(BEFORE ${CAF_INCLUDE_DIRS}) +-set(LINK_LIBS ${LINK_LIBS} ${CAF_LIBRARIES}) +- +-# OpenSSL +-find_package(OpenSSL REQUIRED) +-set(LINK_LIBS ${LINK_LIBS} ${OPENSSL_LIBRARIES}) ++set(LINK_LIBS ${LINK_LIBS} caf::core caf::io caf::openssl) + + # RocksDB + find_package(RocksDB) +Submodule cmake contains modified content +diff --git a/cmake/FindCAF.cmake b/cmake/FindCAF.cmake +index 30d75bd..aebaffa 100644 +--- a/cmake/FindCAF.cmake ++++ b/cmake/FindCAF.cmake +@@ -2,27 +2,34 @@ + # + # Use this module as follows: + # +-# find_package(CAF [COMPONENTS *] [REQUIRED]) ++# find_package(CAF [COMPONENTS *] [REQUIRED]) + # + # Variables used by this module (they can change the default behaviour and need + # to be set before calling find_package): + # +-# CAF_ROOT_DIR Set this variable either to an installation prefix or to wa +-# CAF build directory where to look for the CAF libraries. ++# CAF_ROOT_DIR Set this variable either to an installation prefix or to wa CAF ++# build directory where to look for the CAF libraries. + # + # Variables defined by this module: + # +-# CAF_FOUND System has CAF headers and library +-# CAF_VERSION Found CAF release number +-# CAF_LIBRARIES List of library files for all components +-# CAF_INCLUDE_DIRS List of include paths for all components ++# CAF_FOUND System has CAF headers and library ++# CAF_LIBRARIES List of library files for all components ++# CAF_INCLUDE_DIRS List of include paths for all components ++# CAF_LIBRARY_$C Library file for component $C ++# CAF_INCLUDE_DIR_$C Include path for component $C + +-if(CAF_FIND_COMPONENTS STREQUAL "") ++if (CAF_FIND_COMPONENTS STREQUAL "") + message(FATAL_ERROR "FindCAF requires at least one COMPONENT.") +-endif() ++endif () ++ ++set(suffix "") ++if (NOT BUILD_SHARED_LIBS) ++ set(suffix "_static") ++endif () + + # iterate over user-defined components + foreach (comp ${CAF_FIND_COMPONENTS}) ++ message(STATUS "CAF component: ${comp}") + # we use uppercase letters only for variable names + string(TOUPPER "${comp}" UPPERCOMP) + if ("${comp}" STREQUAL "core") +@@ -34,70 +41,68 @@ foreach (comp ${CAF_FIND_COMPONENTS}) + endif () + if (CAF_ROOT_DIR) + set(header_hints +- "${CAF_ROOT_DIR}/include" +- "${CAF_ROOT_DIR}/libcaf_${comp}" ++ "${CAF_ROOT_DIR}/include" "${CAF_ROOT_DIR}/libcaf_${comp}" + "${CAF_ROOT_DIR}/../libcaf_${comp}" + "${CAF_ROOT_DIR}/../../libcaf_${comp}") + endif () +- find_path(CAF_INCLUDE_DIR_${UPPERCOMP} +- NAMES +- ${HDRNAME} +- HINTS +- ${header_hints} +- /usr/include +- /usr/local/include +- /opt/local/include +- /sw/include +- ${CMAKE_INSTALL_PREFIX}/include +- ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) ++ find_path( ++ CAF_INCLUDE_DIR_${UPPERCOMP} ++ NAMES ${HDRNAME} ++ HINTS ${header_hints} /usr/include /usr/local/include /opt/local/include ++ /sw/include ${CMAKE_INSTALL_PREFIX}/include) + mark_as_advanced(CAF_INCLUDE_DIR_${UPPERCOMP}) +- if (NOT "${CAF_INCLUDE_DIR_${UPPERCOMP}}" +- STREQUAL "CAF_INCLUDE_DIR_${UPPERCOMP}-NOTFOUND") ++ message(STATUS "is ${CAF_INCLUDE_DIR_${UPPERCOMP}}") ++ if (NOT "${CAF_INCLUDE_DIR_${UPPERCOMP}}" STREQUAL ++ "CAF_INCLUDE_DIR_${UPPERCOMP}-NOTFOUND") + # mark as found (set back to false when missing library or build header) + set(CAF_${comp}_FOUND true) + # check for CMake-generated build header for the core component + if ("${comp}" STREQUAL "core") +- find_path(caf_build_header_path +- NAMES +- caf/detail/build_config.hpp +- HINTS +- ${header_hints} +- /usr/include +- /usr/local/include +- /opt/local/include +- /sw/include +- ${CMAKE_INSTALL_PREFIX}/include +- ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) ++ # read content of config.hpp ++ file(READ "${CAF_INCLUDE_DIR_CORE}/caf/config.hpp" CONFIG_HPP) ++ # get line containing the version ++ string(REGEX MATCH "#define CAF_VERSION [0-9]+" VERSION_LINE ++ "${CONFIG_HPP}") ++ # extract version number from line ++ string(REGEX MATCH "[0-9]+" VERSION_INT "${VERSION_LINE}") ++ # calculate major, minor, and patch version ++ math(EXPR CAF_VERSION_MAJOR "${VERSION_INT} / 10000") ++ math(EXPR CAF_VERSION_MINOR "( ${VERSION_INT} / 100) % 100") ++ math(EXPR CAF_VERSION_PATCH "${VERSION_INT} % 100") ++ # create full version string ++ set(CAF_VERSION ++ "${CAF_VERSION_MAJOR}.${CAF_VERSION_MINOR}.${CAF_VERSION_PATCH}") ++ find_path( ++ caf_build_header_path ++ NAMES caf/detail/build_config.hpp ++ HINTS ${header_hints} /usr/include /usr/local/include ++ /opt/local/include /sw/include ${CMAKE_INSTALL_PREFIX}/include) + if ("${caf_build_header_path}" STREQUAL "caf_build_header_path-NOTFOUND") + message(WARNING "Found all.hpp for CAF core, but not build_config.hpp") + set(CAF_${comp}_FOUND false) +- else() +- list(APPEND CAF_INCLUDE_DIRS "${caf_build_header_path}") +- endif() +- endif() ++ else () ++ list(APPEND CAF_INCLUDE_DIR_${UPPERCOMP} "${caf_build_header_path}") ++ endif () ++ endif () ++ message(STATUS "CAF component: ${comp} found") + list(APPEND CAF_INCLUDE_DIRS "${CAF_INCLUDE_DIR_${UPPERCOMP}}") + # look for (.dll|.so|.dylib) file, again giving hints for non-installed CAFs + # skip probe_event as it is header only + if (NOT ${comp} STREQUAL "probe_event" AND NOT ${comp} STREQUAL "test") + if (CAF_ROOT_DIR) + set(library_hints "${CAF_ROOT_DIR}/lib") ++ message(STATUS "library_hints: ${library_hints}") + endif () +- find_library(CAF_LIBRARY_${UPPERCOMP} +- NAMES +- "caf_${comp}" +- "caf_${comp}_static" +- HINTS +- ${library_hints} +- /usr/lib +- /usr/local/lib +- /opt/local/lib +- /sw/lib +- ${CMAKE_INSTALL_PREFIX}/lib +- ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR} +- ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_BUILD_TYPE}) ++ find_library( ++ CAF_LIBRARY_${UPPERCOMP} ++ NAMES "caf_${comp}${suffix}" ++ HINTS ${library_hints} /usr/lib /usr/local/lib /opt/local/lib /sw/lib ++ ${CMAKE_INSTALL_PREFIX}/lib) ++ message(STATUS "CAF component lib: ${comp}${suffix}") + mark_as_advanced(CAF_LIBRARY_${UPPERCOMP}) +- if ("${CAF_LIBRARY_${UPPERCOMP}}" +- STREQUAL "CAF_LIBRARY_${UPPERCOMP}-NOTFOUND") ++ message(STATUS "${CAF_LIBRARY_${UPPERCOMP}}") ++ if ("${CAF_LIBRARY_${UPPERCOMP}}" STREQUAL ++ "CAF_LIBRARY_${UPPERCOMP}-NOTFOUND") + set(CAF_${comp}_FOUND false) + else () + set(CAF_LIBRARIES ${CAF_LIBRARIES} ${CAF_LIBRARY_${UPPERCOMP}}) +@@ -108,37 +113,81 @@ endforeach () + + if (DEFINED CAF_INCLUDE_DIRS) + list(REMOVE_DUPLICATES CAF_INCLUDE_DIRS) +-endif() +- +- +-if (NOT CAF_INCLUDE_DIR_CORE STREQUAL "CAF_INCLUDE_DIR_CORE-NOTFOUND") +- # read content of config.hpp +- file(READ "${CAF_INCLUDE_DIR_CORE}/caf/config.hpp" CONFIG_HPP) +- # get line containing the version +- string(REGEX MATCH "#define CAF_VERSION [0-9]+" VERSION_LINE "${CONFIG_HPP}") +- # extract version number from line +- string(REGEX MATCH "[0-9]+" VERSION_INT "${VERSION_LINE}") +- # calculate major, minor, and patch version +- math(EXPR CAF_VERSION_MAJOR "${VERSION_INT} / 10000") +- math(EXPR CAF_VERSION_MINOR "( ${VERSION_INT} / 100) % 100") +- math(EXPR CAF_VERSION_PATCH "${VERSION_INT} % 100") +- # create full version string +- set(CAF_VERSION "${CAF_VERSION_MAJOR}.${CAF_VERSION_MINOR}.${CAF_VERSION_PATCH}") +- if (NOT CAF_VERSION) +- unset(CAF_VERSION) +- message(WARNING "Unable to determine CAF version") +- endif () + endif () + + # let CMake check whether all requested components have been found + include(FindPackageHandleStandardArgs) +-find_package_handle_standard_args(CAF +- FOUND_VAR CAF_FOUND +- REQUIRED_VARS CAF_VERSION CAF_LIBRARIES CAF_INCLUDE_DIRS +- HANDLE_COMPONENTS) ++find_package_handle_standard_args( ++ CAF ++ REQUIRED_VARS ++ CAF_LIBRARIES ++ CAF_INCLUDE_DIRS ++ VERSION_VAR ++ CAF_VERSION ++ HANDLE_COMPONENTS) ++ ++if (NOT CAF_FOUND) ++ unset(CAF_LIBRARIES) ++ unset(CAF_INCLUDE_DIRS) ++endif () + + # final step to tell CMake we're done +-mark_as_advanced(CAF_ROOT_DIR +- CAF_VERSION +- CAF_LIBRARIES +- CAF_INCLUDE_DIRS) ++mark_as_advanced(CAF_ROOT_DIR CAF_LIBRARIES CAF_INCLUDE_DIRS) ++ ++if (CAF_FOUND) ++ include(CMakeFindDependencyMacro) ++ message(STATUS "CREATING CAF TARGETS") ++ if (CAF_core_FOUND AND NOT TARGET caf::core) ++ add_library(caf::core UNKNOWN IMPORTED GLOBAL) ++ set_target_properties( ++ caf::core ++ PROPERTIES ++ IMPORTED_LOCATION "${CAF_LIBRARY_CORE}" INTERFACE_INCLUDE_DIRECTORIES ++ "${CAF_INCLUDE_DIR_CORE}") ++ endif () ++ if (CAF_io_FOUND AND NOT TARGET caf::io) ++ find_dependency(Threads) ++ add_library(caf::io UNKNOWN IMPORTED GLOBAL) ++ set_target_properties( ++ caf::io ++ PROPERTIES ++ IMPORTED_LOCATION "${CAF_LIBRARY_IO}" ++ INTERFACE_INCLUDE_DIRECTORIES "${CAF_INCLUDE_DIR_IO}" ++ INTERFACE_LINK_LIBRARIES "caf::core;Threads::Threads") ++ endif () ++ if (CAF_openssl_FOUND AND NOT TARGET caf::openssl) ++ add_library(caf::openssl UNKNOWN IMPORTED GLOBAL) ++ set_target_properties( ++ caf::openssl ++ PROPERTIES ++ IMPORTED_LOCATION "${CAF_LIBRARY_OPENSSL}" ++ INTERFACE_INCLUDE_DIRECTORIES "${CAF_INCLUDE_DIR_OPENSSL}" ++ INTERFACE_LINK_LIBRARIES "caf::core;caf::io") ++ if (NOT BUILD_SHARED_LIBS) ++ set(OPENSSL_USE_STATIC_LIBS TRUE) ++ find_dependency(OpenSSL) ++ set_property( ++ TARGET caf::openssl ++ APPEND ++ PROPERTY INTERFACE_LINK_LIBRARIES "OpenSSL::SSL") ++ endif () ++ endif () ++ if (CAF_opencl_FOUND AND NOT TARGET caf::opencl) ++ add_library(caf::opencl UNKNOWN IMPORTED GLOBAL) ++ set_target_properties( ++ caf::opencl ++ PROPERTIES ++ IMPORTED_LOCATION "${CAF_LIBRARY_OPENCL}" ++ INTERFACE_INCLUDE_DIRECTORIES "${CAF_INCLUDE_DIR_OPENCL}" ++ INTERFACE_LINK_LIBRARIES "caf::core") ++ endif () ++ if (CAF_test_FOUND AND NOT TARGET caf::test) ++ add_library(caf::test INTERFACE IMPORTED GLOBAL) ++ set_target_properties( ++ caf::test ++ PROPERTIES ++ INTERFACE_INCLUDE_DIRECTORIES "${CAF_INCLUDE_DIR_TEST}" ++ ++ INTERFACE_LINK_LIBRARIES "caf::core") ++ endif () ++endif () diff --git a/nix/broker/source.json b/nix/broker/source.json new file mode 100644 index 00000000000..b758aa28b21 --- /dev/null +++ b/nix/broker/source.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/zeek/broker", + "rev": "e65cca620b1d2d26af6c5a9ee9513321fc3ed2a0", + "date": "2020-03-09T13:05:12-07:00", + "path": "/nix/store/18cmr8qkiy04846pnn5p2h67j2b18yjn-broker-e65cca6", + "sha256": "0c5v69afv5ip0bp385x5cvww1999lhrvvyjk52khb7j4bimx05wq", + "fetchSubmodules": true, + "deepClone": false, + "leaveDotGit": false +} diff --git a/nix/caf/source.json b/nix/caf/source.json new file mode 100644 index 00000000000..fc635bbc663 --- /dev/null +++ b/nix/caf/source.json @@ -0,0 +1,7 @@ +{ + "owner": "actor-framework", + "repo": "actor-framework", + "rev": "22ccaba0580eee14dd5de01b037a20a2c9f4df8e", + "sha256": "1w7sanxhh4sv4ap3ildqzlziazcy7ydx2pbjjrf3k60lcwq0lzn6", + "fetchSubmodules": false +} \ No newline at end of file diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 00000000000..1ac30b6aa89 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,10 @@ +let + nixpkgs_ = import ./pinned.nix; + vastPkgs = import ./overlay.nix; +in + +{ nixpkgs ? nixpkgs_ }: +(import nixpkgs { config = {}; overlays = [ vastPkgs ]; }) // +{ + inherit nixpkgs_; +} diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 00000000000..6087b6eb04d --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,39 @@ +final: prev: +let + inherit (final) lib; + static_stdenv = final.stdenv.hostPlatform.isMusl; +in { + nix-gitDescribe = final.callPackage ./gitDescribe.nix {}; + arrow-cpp = prev.arrow-cpp.overrideAttrs (old: { + patches = old.patches ++ lib.optional static_stdenv arrow/fix-static-jemalloc.patch; + }); + caf = let + source = builtins.fromJSON (builtins.readFile ./caf/source.json); + in prev.caf.overrideAttrs (old: { + # fetchFromGitHub uses ellipsis in the parameter set to be hash method + # agnostic. Because of that, callPackageWith does not detect that sha256 + # is a required argument, and it has to be passed explicitly instead. + src = lib.callPackageWith source final.fetchFromGitHub { inherit (source) sha256; }; + } // lib.optionalAttrs static_stdenv { + cmakeFlags = old.cmakeFlags ++ [ + "-DCAF_BUILD_STATIC=ON" + "-DCAF_BUILD_STATIC_ONLY=ON" + "-DOPENSSL_USE_STATIC_LIBS=TRUE" + "-DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=ON" + "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW" + ]; + hardeningDisable = [ + "pic" + ]; + # Building statically implies using -flto. Since we produce a final binary with + # link time optimizaitons in VAST, we need to make sure that type definitions that + # are parsed in both projects are the same, otherwise the compiler will complain + # at the optimization stage. + # TODO: Remove when updating to CAF 0.18. + NIX_CFLAGS_COMPILE = "-std=c++17"; + dontStrip = true; + }); + broker = final.callPackage ./broker {python = final.python3;}; + vast-source = final.nix-gitignore.gitignoreSource [] ./..; + vast = final.callPackage ./vast {}; +} diff --git a/nix/pinned.nix b/nix/pinned.nix new file mode 100644 index 00000000000..4fcfcd90f22 --- /dev/null +++ b/nix/pinned.nix @@ -0,0 +1,9 @@ +# See https://nixos.wiki/wiki/FAQ/Pinning_Nixpkgs for more information on pinning +builtins.fetchTarball { + # Descriptive name to make the store path easier to identify + name = "nixpkgs-unstable-2020-06-07"; + # Commit hash for nixpkgs-unstable as of date + url = https://github.com/NixOS/nixpkgs-channels/archive/029a5de08390bb03c3f44230b064fd1850c6658a.tar.gz; + # Hash obtained using `nix-prefetch-url --unpack ` + sha256 = "03fjkzhrs2avcvdabgm7a65rnyjaqbqdnv4q86qyjkkwg64g5m8x"; +} diff --git a/nix/static-binary.sh b/nix/static-binary.sh new file mode 100755 index 00000000000..e0c64fea503 --- /dev/null +++ b/nix/static-binary.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p git nix coreutils nix-prefetch-github + +dir="$(dirname "$(readlink -f "$0")")" +toplevel="$(git -C ${dir} rev-parse --show-toplevel)" +desc="$(git -C ${dir} describe)" +vast_rev="$(git -C "${toplevel}" rev-parse HEAD)" + +target="${STATIC_BINARY_TARGET:-vast}" + +if [ "$1" == "--use-head" ]; then + source_json="$(nix-prefetch-github --rev=${vast_rev} tenzir vast)" + read -r -d '' exp < ${dir}/caf/source.json +nix-prefetch-git --quiet --rev ${broker_rev} --fetch-submodules https://github.com/zeek/broker \ + > ${dir}/broker/source.json diff --git a/nix/vast.nix b/nix/vast/default.nix similarity index 91% rename from nix/vast.nix rename to nix/vast/default.nix index 09d60e1b37d..bb531558771 100644 --- a/nix/vast.nix +++ b/nix/vast/default.nix @@ -1,5 +1,6 @@ { stdenv , lib +, vast-source , nix-gitignore , nix-gitDescribe , cmake @@ -9,6 +10,7 @@ , caf , libpcap , arrow-cpp +, broker , zstd , jemalloc , python3Packages @@ -31,7 +33,7 @@ let ] ); - src = nix-gitignore.gitignoreSource [ "/nix" ] ../.; + src = vast-source; version = if (versionOverride != null) then versionOverride else stdenv.lib.fileContents (nix-gitDescribe src); in @@ -48,7 +50,7 @@ stdenv.mkDerivation rec { nativeBuildInputs = [ cmake ]; propagatedNativeBuildInputs = [ pkgconfig pandoc ]; - buildInputs = [ libpcap jemalloc ]; + buildInputs = [ libpcap jemalloc broker ]; propagatedBuildInputs = [ arrow-cpp caf ]; cmakeFlags = [ @@ -56,10 +58,12 @@ stdenv.mkDerivation rec { "-DVAST_RELOCATABLE_INSTALL=OFF" "-DVAST_VERSION_TAG=${version}" "-DVAST_USE_JEMALLOC=ON" + "-DBROKER_ROOT_DIR=${broker}" # gen-table-slices runs at build time "-DCMAKE_SKIP_BUILD_RPATH=OFF" ] ++ lib.optionals static [ "-DVAST_STATIC_EXECUTABLE:BOOL=ON" + "-DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=ON" "-DZSTD_ROOT=${zstd}" ] ++ lib.optional disableTests "-DBUILD_UNIT_TESTS=OFF"; diff --git a/scripts/trigger-action b/scripts/trigger-action new file mode 100755 index 00000000000..3c9c32a0e78 --- /dev/null +++ b/scripts/trigger-action @@ -0,0 +1,22 @@ +#!/bin/sh + +# e - non-interactive terminals will exit on failures immediately +set -e + +usage() { + echo "usage: $(basename "$0") []" + echo + echo " A GitHub personal access token with \"Repo\" access must be" + echo " availabe in the environment variable GITHUB_TOKEN." + echo +} + +if [ "$#" -lt 1 ]; then + usage >&2 + exit 1 +fi + +curl -X POST https://api.github.com/repos/tenzir/vast/dispatches \ + -H "Accept: application/vnd.github.everest-preview+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + --data "{\"event_type\": \"$1\", \"client_payload\": ${2:-{\}}}" diff --git a/scripts/trigger-static-build b/scripts/trigger-static-build new file mode 100755 index 00000000000..d10ac366455 --- /dev/null +++ b/scripts/trigger-static-build @@ -0,0 +1,17 @@ +#!/bin/sh + +# e - non-interactive terminals will exit on failures immediately +set -e + +# Usage: ./trigger-static-build [] [] + +if [ "$#" -lt 1 ]; then + >&2 echo "Using 'master' as default branch." +fi + +if [ -z "${GITHUB_TOKEN}" ]; then + >&2 echo "A GitHub personal access token with \"Repo\" access must be" + >&2 echo "availabe in the environment variable GITHUB_TOKEN." +fi + +./trigger-action static-binary "{\"ref\": \"${1:-master}\", \"args\": \"${2}\"}" diff --git a/shell.nix b/shell.nix index 9330e893cd9..cb95638f1f2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,7 +1,22 @@ -{ pkgs ? import {} }: -((import ./default.nix { inherit pkgs; }).override{ - versionOverride = "dev"; -}).overrideAttrs (old: { - src = null; - hardeningDisable = (old.hardeningDisable or []) ++ [ "fortify" ]; +{ pkgs ? import ./nix { }, useClang ? false }: +let + inherit (pkgs) lib; + llvmPkgs = pkgs.buildPackages.llvmPackages_10; + stdenv = if useClang then llvmPkgs.libcxxStdenv else pkgs.stdenv; + static_stdenv = stdenv.hostPlatform.isMusl; + mkShell = pkgs.mkShell.override { inherit stdenv; }; +in +mkShell ({ + name = "vast-dev-" + (if useClang then "clang" else "gcc"); + hardeningDisable = [ "fortify" ] ++ lib.optional static_stdenv "pic"; + inputsFrom = [ pkgs.vast ]; + shellHook = '' + echo "Entering Event Horizon environment" + ''; +} // lib.optionalAttrs static_stdenv { + VAST_STATIC_EXECUTABLE = "ON"; + ZSTD_ROOT = "${pkgs.zstd}"; +} // lib.optionalAttrs (stdenv.isLinux && !static_stdenv) { + nativeBuildInputs = [ llvmPkgs.lldClang.bintools ]; + NIX_CFLAGS_LINK = "-fuse-ld=lld"; })