Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add a static binary workflow #966

Merged
merged 26 commits into from Jul 7, 2020
Merged

Add a static binary workflow #966

Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
e3232b1
Extend shell.nix to allow toolchain selection
tobim Jul 3, 2020
432f107
Allow passing ZSTD_ROOT as an env variable
tobim Jul 4, 2020
d9e5150
Fix a print statement in Findjemalloc
tobim Jul 4, 2020
3e4709d
Read VAST_STATIC_EXECUTABLE from env as fallback
tobim Jul 4, 2020
d690de4
Use jemalloc by default in static builds
tobim Jul 4, 2020
897befb
Add pkgStatic support to the nix scaffold
tobim Jul 4, 2020
372d1ea
Make vast source an explicit attribute
tobim Jul 5, 2020
929beb8
Optionally use the rev from origin
tobim Jul 6, 2020
bf4e2b8
Add /nix/build/ to the git ignore list
tobim Jul 6, 2020
b14a674
Add a static binary workflow
tobim Jul 6, 2020
b132bff
Use vast as default target
tobim Jul 6, 2020
580609e
Clean up static binary workflow jobs
tobim Jul 6, 2020
c09322e
Add scripts to trigger the static build
tobim Jul 6, 2020
4fa59f2
Simplify static workflow job conditionals
tobim Jul 6, 2020
487d6f4
Fix static binary tarball creation
tobim Jul 6, 2020
01bc025
Explain why c++17 is needed for static CAF
tobim Jul 6, 2020
730a410
Unify static binary jobs
tobim Jul 6, 2020
11986e9
Add a "VAST Static" badge to the readme
tobim Jul 6, 2020
8fad8d5
Fix Static binary target name
tobim Jul 6, 2020
01d5180
Use a constant name for the static release asset
tobim Jul 6, 2020
faa91cf
Make the release artifact names stable
tobim Jul 6, 2020
2847264
Add a changelog entry for the static binary workflow
tobim Jul 6, 2020
46d5de9
Change the YAML filename extention to .yaml
tobim Jul 6, 2020
87622de
Fix formatting in the changelog entry
tobim Jul 6, 2020
73cfb4f
Fix the release asset name
tobim Jul 6, 2020
77a66dc
Fix self modification trigger in static workflow
tobim Jul 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
69 changes: 69 additions & 0 deletions .github/workflows/static-binary.yml
View file
Open in desktop
@@ -0,0 +1,69 @@
name: "VAST Static"
on:
repository_dispatch:
types: static-binary
push:
branches:
- master
pull_request:
paths:
- '.github/workflows/static-binary.yml'
dominiklohmann marked this conversation as resolved.
Show resolved Hide resolved
release:
types: published
tobim marked this conversation as resolved.
Show resolved Hide resolved

jobs:
static_binary:
runs-on: ubuntu-latest
name: Static Binary
env:
BUILD_DIR: build
STATIC_BINARY_TARGET: ${{ github.event.client_payload.args == '' && 'vast' || github.event.client_payload.args }}
steps:
- name: Checkout
if: github.event_name == 'repository_dispatch'
uses: actions/checkout@v1
with:
ref: ${{ github.event.client_payload.ref == '' && 'master' || github.event.client_payload.ref }}

- name: Checkout
if: github.event_name != 'repository_dispatch'
uses: actions/checkout@v1

- name: Install Nix
uses: cachix/install-nix-action@v10

- name: Setup Cachix
uses: cachix/cachix-action@v6
with:
name: vast
signingKey: '${{ secrets.CACHIX_VAST_SIGNING_KEY }}'

- name: Build a Static Binary
run: |
nix/static-binary.sh --use-head

- name: Create Paths
id: create_paths
run: |
ARTIFACT_NAME=$(ls "${BUILD_DIR}" | grep "vast.*.tar.gz")
echo "::set-output name=artifact_name::${ARTIFACT_NAME}"

- name: Upload Artifact to Github
uses: actions/upload-artifact@v1
with:
name: "${{ steps.create_paths.outputs.artifact_name }}"
path: "${{ env.BUILD_DIR }}/${{ steps.create_paths.outputs.artifact_name }}"

- name: Publish to GitHub Release
if: github.event_name == 'release' && github.event.action == 'published'
uses: actions/upload-release-asset@v1.0.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ github.event.release.upload_url }}
asset_path: "${{ env.BUILD_DIR }}/${{ steps.create_paths.outputs.artifact_name }}"
# The asset name is constant so we can permanently link to
# https://github.com/tenzir/vast/releases/latest/download/vast-linux-static.tar.gz
# for a build of the latest release.
asset_name: "vast-linux-static.tar.gz"
asset_content_type: application/gzip
tobim marked this conversation as resolved.
Show resolved Hide resolved
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Expand Up @@ -5,6 +5,7 @@

# Build artifacts
/build*/
/nix/build/
/compile_commands.json
.mypy_cache/
__pycache__/
Expand Down
4 changes: 2 additions & 2 deletions CMakeLists.txt
View file
Open in desktop
Expand Up @@ -82,8 +82,8 @@ option(VAST_ENABLE_ASSERTIONS "Enable Assertions"
option(VAST_RELOCATABLE_INSTALL "Enable relocatable installations" ON)
option(VAST_USE_BUNDLED_CAF "Always use the CAF submodule" OFF)
option(ENABLE_ZEEK_TO_VAST "Build zeek-to-vast" ON)
option(VAST_STATIC_EXECUTABLE "Link VAST statically")
option(VAST_USE_JEMALLOC "Use jemalloc instead of libc malloc" OFF)
option(VAST_STATIC_EXECUTABLE "Link VAST statically" $ENV{VAST_STATIC_EXECUTABLE})
option(VAST_USE_JEMALLOC "Use jemalloc instead of libc malloc" "${VAST_STATIC_EXECUTABLE}")
Comment on lines +85 to +86
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed by mistake? This doesn't look right to me.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The intention is to use the environment variable if the option has not been given explicitly. The order of precedence is now:
explicit option > environment > default (implicitly off because the empty string is treated as False).

The reason I changed these is that I can use nix to set up an environment for a static build like so:

nix-shell -E 'let pkgs = import ./nix { }; in import ./shell.nix {pkgs = pkgs.pkgsStatic;}'

The toolchain in this environment is not capable to produce shared libraries, so one has to pass several options to CMake to get a working build tree with a configuration similar to the static build in CI:

cmake -Bbuild -S . -DVAST_STATIC_EXECUTABLE=ON -DZSTD_ROOT="$(nix-build ./nix --no-out-link -A pkgsStatic.zstd)" -DVAST_USE_JEMALLOC=ON

With the changes to CMake adding these options is not required, while the regular build is not affected.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I really don't like how inconsistent this is. Can you create a follow-up story for a) prefixing all CMake variables that the user can set with VAST_, and b) for exposing them on the command line like this?

cmake_dependent_option(
BUILD_SHARED_LIBS "Build shared libraries instead of static" ON
"NOT VAST_STATIC_EXECUTABLE" OFF)
Expand Down
9 changes: 6 additions & 3 deletions README.md
View file
Open in desktop
Expand Up @@ -10,8 +10,9 @@
The network telemetry engine for data-driven security investigations.

[![Build Status][ci-badge]][ci-url]
[![Static Build Status][ci-static-badge]][ci-static-url]
[![Examples Status][ci-examples-badge]][ci-examples-url]
[![Docker Status][docker-badge]][ci-docker-url]
[![Docker Status][ci-docker-badge]][ci-docker-url]
[![Changelog][changelog-badge]][changelog-url]
[![Latest Release][latest-release-badge]][latest-release-url]
[![Chat][chat-badge]][chat-url]
Expand Down Expand Up @@ -136,10 +137,12 @@ proceedings][nsdi-proceedings].
[chat-url]: https://gitter.im/tenzir/chat
[ci-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3AVAST
[ci-badge]: https://github.com/tenzir/vast/workflows/VAST/badge.svg?branch=master
[ci-examples-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3AJupyter+Notebook
[ci-examples-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3A%22Jupyter+Notebook%22
[ci-examples-badge]: https://github.com/tenzir/vast/workflows/Jupyter%20Notebook/badge.svg?branch=master
[ci-docker-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3AVAST+Docker
[ci-docker-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3A%22VAST+Docker%22
[ci-docker-badge]: https://github.com/tenzir/vast/workflows/VAST%20Docker/badge.svg?branch=master
[ci-static-url]: https://github.com/tenzir/vast/actions?query=branch%3Amaster+workflow%3A%22VAST+Static%22
[ci-static-badge]: https://github.com/tenzir/vast/workflows/VAST%20Static/badge.svg?branch=master
[license-badge]: https://img.shields.io/badge/license-BSD-blue.svg
[license-url]: https://raw.github.com/vast-io/vast/master/COPYING
[changelog-badge]: https://img.shields.io/badge/view-changelog-green.svg
Expand Down
7 changes: 7 additions & 0 deletions cmake/FindZSTD.cmake
View file
Open in desktop
Expand Up @@ -23,6 +23,13 @@ set(ZSTD_STATIC_LIB_NAME
${CMAKE_STATIC_LIBRARY_PREFIX}zstd${ZSTD_STATIC_LIB_SUFFIX})

# First, find via if specified ZTD_ROOT
if (NOT ZSTD_ROOT)
# We look for an environment variable of the same name too, because
# that can be provided by a tool like nix-shell or docker. Wo do this
# so the user doesn't have to remember to add "-DZSTD_ROOT=..." to their
# cmake invocations.
Comment on lines +27 to +30
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Having this for some variables but not all is really confusing and makes maintenance of the CMake way harder. I'm already always afraid to change CMake code because it might break the Nix build.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is essentially a workaround for a bug in the generated .pc files in ZSTD. We can probably get rid of FindZSTD.cmake when upgrading to the next Arrow release thanks to apache/arrow#7388.

set(ZSTD_ROOT "$ENV{ZSTD_ROOT}")
endif ()
if (ZSTD_ROOT)
message(STATUS "Using ZSTD_ROOT: ${ZSTD_ROOT}")
find_library(
Expand Down
2 changes: 1 addition & 1 deletion cmake/Findjemalloc.cmake
View file
Open in desktop
Expand Up @@ -7,7 +7,7 @@ else ()
endif ()

if (jemalloc_ROOT_DIR)
message(STATUS "Using jemalloc_ROOT: ${jemalloc_ROOT}")
message(STATUS "Using jemalloc_ROOT: ${jemalloc_ROOT_DIR}")
find_path(
jemalloc_INCLUDE_DIR
NAMES jemalloc/jemalloc.h
Expand Down
67 changes: 67 additions & 0 deletions nix/arrow/fix-static-jemalloc.patch
View file
Open in desktop
@@ -0,0 +1,67 @@
diff --git a/cmake_modules/ArrowJemallocTarget.cmake.in b/cmake_modules/ArrowJemallocTarget.cmake.in
new file mode 100644
index 000000000..123fdf4b7
--- /dev/null
+++ b/cmake_modules/ArrowJemallocTarget.cmake.in
@@ -0,0 +1,18 @@
+
+# Compute the installation prefix relative to this file.
+get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH)
+get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH)
+get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH)
+get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH)
+if(_IMPORT_PREFIX STREQUAL "/")
+ set(_IMPORT_PREFIX "")
+endif()
+
+add_library(jemalloc::jemalloc STATIC IMPORTED)
+set_target_properties(jemalloc::jemalloc
+ PROPERTIES INTERFACE_LINK_LIBRARIES
+ Threads::Threads)
+set_property(TARGET jemalloc::jemalloc APPEND PROPERTY IMPORTED_CONFIGURATIONS @CMAKE_BUILD_TYPE@)
+set_target_properties(jemalloc::jemalloc PROPERTIES
+ IMPORTED_LINK_INTERFACE_LANGUAGES_@CMAKE_BUILD_TYPE@ "C;CXX"
+ IMPORTED_LOCATION_@CMAKE_BUILD_TYPE@ "${_IMPORT_PREFIX}/lib/arrow/@JEMALLOC_STATIC_NAME@")
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index ba3138d20..2722968fd 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -1308,8 +1308,9 @@ if(ARROW_JEMALLOC)
set(ARROW_JEMALLOC_USE_SHARED OFF)
set(JEMALLOC_PREFIX
"${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
+ set(JEMALLOC_STATIC_NAME "libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(JEMALLOC_STATIC_LIB
- "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ "${JEMALLOC_PREFIX}/lib/${JEMALLOC_STATIC_NAME}")
set(JEMALLOC_CONFIGURE_COMMAND ./configure "AR=${CMAKE_AR}" "CC=${CMAKE_C_COMPILER}")
if(CMAKE_OSX_SYSROOT)
list(APPEND JEMALLOC_CONFIGURE_COMMAND "SDKROOT=${CMAKE_OSX_SYSROOT}")
@@ -1353,6 +1354,14 @@ if(ARROW_JEMALLOC)
INTERFACE_INCLUDE_DIRECTORIES
"${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src")
add_dependencies(jemalloc::jemalloc jemalloc_ep)
+ if(ARROW_BUILD_STATIC)
+ install(FILES "${JEMALLOC_STATIC_LIB}"
+ DESTINATION "${CMAKE_INSTALL_LIBDIR}/arrow")
+ configure_file("${PROJECT_SOURCE_DIR}/cmake_modules/ArrowJemallocTarget.cmake.in"
+ "${CMAKE_CURRENT_BINARY_DIR}/ArrowJemallocTarget.cmake" @ONLY)
+ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ArrowJemallocTarget.cmake"
+ DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/arrow")
+ endif()
endif()

# ----------------------------------------------------------------------
diff --git a/src/arrow/ArrowConfig.cmake.in b/src/arrow/ArrowConfig.cmake.in
index 0e595066d..c52e3cb07 100644
--- a/src/arrow/ArrowConfig.cmake.in
+++ b/src/arrow/ArrowConfig.cmake.in
@@ -39,5 +39,8 @@ include("${CMAKE_CURRENT_LIST_DIR}/ArrowOptions.cmake")
# Load targets only once. If we load targets multiple times, CMake reports
# already existent target error.
if(NOT (TARGET arrow_shared OR TARGET arrow_static))
+ if(ARROW_BUILD_STATIC AND ARROW_JEMALLOC)
+ include("${CMAKE_CURRENT_LIST_DIR}/ArrowJemallocTarget.cmake")
+ endif()
include("${CMAKE_CURRENT_LIST_DIR}/ArrowTargets.cmake")
endif()
47 changes: 47 additions & 0 deletions nix/broker/default.nix
View file
Open in desktop
@@ -0,0 +1,47 @@
{ stdenv, lib, fetchgit, cmake, caf, openssl, python, ncurses
, static ? stdenv.hostPlatform.isMusl
, linkTimeOptimization ? static }:

let
source = builtins.fromJSON (builtins.readFile ./source.json);
isCross = stdenv.buildPlatform != stdenv.hostPlatform;
fixODR = static && linkTimeOptimization;
in

stdenv.mkDerivation rec {
pname = "broker";
version = builtins.substring 0 10 source.date;

src = lib.callPackageWith source fetchgit {};

nativeBuildInputs = [ cmake ];
buildInputs = [ caf openssl ]
++ lib.optionals (!static) [ python ncurses ];

cmakeFlags = [
"-DBUILD_SHARED_LIBS=ON"
"-DBROKER_DISABLE_DOCS=ON"
"-DCAF_ROOT_DIR=${caf}"
"-DPY_MOD_INSTALL_DIR=${placeholder "out"}/${python.sitePackages}"
] ++ lib.optionals static [
"-DENABLE_STATIC_ONLY=ON"
"-DOPENSSL_USE_STATIC_LIBS=TRUE"
] ++ lib.optionals linkTimeOptimization [
"-DCMAKE_POLICY_DEFAULT_CMP0069=NEW"
"-DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=ON"
] ++ lib.optionals isCross [
"-DBROKER_DISABLE_TESTS=ON"
];
hardeningDisable = lib.optional static "pic";
dontStrip = static;

patches = [ ./fix_static_linkage.patch ];

meta = with lib; {
description = "Zeek networking layer";
homepage = http://zeek.io/;
license = licenses.bsd3;
platforms = platforms.unix;
maintainers = with maintainers; [ tobim ];
};
}