From 8214b445ad6a8c130f48040f3db8314a213e4e3d Mon Sep 17 00:00:00 2001 From: Casey Williams Date: Thu, 26 Jul 2018 10:15:04 -0700 Subject: [PATCH 1/2] (PA-2055) Update cpp-hocon to c710b8cd407339f7f6f3215028fc51cf50695f06 This revision of cpp-hocon ignores some warnings generated by Boost.Variant headers in boost 1.67. --- configs/components/cpp-hocon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/components/cpp-hocon.json b/configs/components/cpp-hocon.json index a2ee4969d5..397df7d772 100644 --- a/configs/components/cpp-hocon.json +++ b/configs/components/cpp-hocon.json @@ -1 +1 @@ -{"url": "git://github.com/puppetlabs/cpp-hocon.git", "ref": "refs/tags/0.1.5"} +{"url": "git://github.com/puppetlabs/cpp-hocon.git", "ref": "c710b8cd407339f7f6f3215028fc51cf50695f06"} From 392bcd7bcb60e1404200bc09029c24f26bfebf59 Mon Sep 17 00:00:00 2001 From: Casey Williams Date: Thu, 26 Jul 2018 10:26:20 -0700 Subject: [PATCH 2/2] (PA-2055) Use boost and yaml-cpp from puppet-runtime Previously, boost and yaml-cpp were provided by separate packages, whether that meant using pl-build-tools (pl-boost and pl-yaml-cpp) or equivalent system packages. This set of changes updates the configuration of C++ components to rely on copies of boost (version 1.67) and yaml-cpp from puppet-runtime instead. It also links these libraries dynamically instead of statically in the agent. --- README.md | 12 ++++---- configs/components/cpp-hocon.rb | 3 +- configs/components/cpp-pcp-client.rb | 2 -- configs/components/facter-precompiled-gem.rb | 15 ++++------ configs/components/facter.rb | 31 +++++++------------- configs/components/leatherman.rb | 11 ++----- configs/components/libwhereami.rb | 4 +-- configs/components/pxp-agent.rb | 28 ++++++++++++++++-- configs/components/runtime.rb | 2 +- 9 files changed, 53 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index a7ea6499ac..ec5174bd12 100644 --- a/README.md +++ b/README.md @@ -61,15 +61,13 @@ If you wish to build puppet-agent or the facter gem yourself: Puppet, you will need to make a few edits in the component and project files. The build process depends on the following packages: - GCC (>=4.8.0) - - Boost (>=1.57) - CMake (>= 3.2.3) - - yaml-cpp (>= 0.5.0) - Any references to pl-gcc, pl-cmake, pl-boost, pl-yaml-cpp, etc. in the - [configs directory](configs/) will need to be changed to refer to - equivalent installable packages on your target operating system. In many - cases, you can drop the `pl-` prefix and ensure that CXX or CC envrionment - variables are what they should be. + Any references to pl-gcc, pl-cmake, etc. in the [configs + directory](configs/) will need to be changed to refer to equivalent + installable packages on your target operating system. In many cases, you + can drop the `pl-` prefix and ensure that CXX or CC environment variables + are what they should be. 4. Update the `location` and `version` in the [puppet-runtime component json file](configs/components/puppet-runtime.json) as follows: - `location` should be a file URL to your local puppet-runtime output diff --git a/configs/components/cpp-hocon.rb b/configs/components/cpp-hocon.rb index f2c527b642..c0870660ff 100644 --- a/configs/components/cpp-hocon.rb +++ b/configs/components/cpp-hocon.rb @@ -22,7 +22,7 @@ special_flags = "-DCMAKE_CXX_FLAGS_RELEASE='-O2 -DNDEBUG'" elsif platform.is_windows? make = "#{settings[:gcc_bindir]}/mingw32-make" - pkg.environment "PATH", "$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" + pkg.environment "PATH", "$(shell cygpath -u #{settings[:prefix]}/lib):$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" pkg.environment "CYGWIN", settings[:cygwin] cmake = "C:/ProgramData/chocolatey/bin/cmake.exe -G \"MinGW Makefiles\"" @@ -45,7 +45,6 @@ -DCMAKE_PREFIX_PATH=#{settings[:prefix]} \ -DCMAKE_INSTALL_PREFIX=#{settings[:prefix]} \ #{special_flags} \ - -DBOOST_STATIC=ON \ ."] end diff --git a/configs/components/cpp-pcp-client.rb b/configs/components/cpp-pcp-client.rb index 1e90783a3e..de01e3fdd8 100644 --- a/configs/components/cpp-pcp-client.rb +++ b/configs/components/cpp-pcp-client.rb @@ -21,7 +21,6 @@ if platform.is_aix? pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-gcc-5.2.0-11.aix#{platform.os_version}.ppc.rpm" pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-cmake-3.2.3-2.aix#{platform.os_version}.ppc.rpm" - pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-yaml-cpp-0.5.1-1.aix#{platform.os_version}.ppc.rpm" # This should be moved to the toolchain file platform_flags = '-DCMAKE_SHARED_LINKER_FLAGS="-Wl,-bbigtoc"' elsif platform.is_macos? @@ -55,7 +54,6 @@ -DCMAKE_INSTALL_PREFIX=#{settings[:prefix]} \ -DCMAKE_INSTALL_RPATH=#{settings[:libdir]} \ -DCMAKE_SYSTEM_PREFIX_PATH=#{settings[:prefix]} \ - -DBOOST_STATIC=ON \ ." ] end diff --git a/configs/components/facter-precompiled-gem.rb b/configs/components/facter-precompiled-gem.rb index 8847ba753b..58bbcf6cea 100644 --- a/configs/components/facter-precompiled-gem.rb +++ b/configs/components/facter-precompiled-gem.rb @@ -1,22 +1,17 @@ component "facter-precompiled-gem" do |pkg, settings, platform| + pkg.build_requires 'puppet-runtime' # provides boost and yaml-cpp pkg.build_requires 'facter-source-gem' pkg.add_source("file://resources/files/facter-gem/facter-precompiled.gemspec.erb") if platform.is_osx? pkg.build_requires "cmake" - pkg.build_requires "boost" - pkg.build_requires "yaml-cpp" elsif platform.is_windows? pkg.build_requires "cmake" pkg.build_requires "pl-toolchain-#{platform.architecture}" - pkg.build_requires "pl-boost-#{platform.architecture}" - pkg.build_requires "pl-yaml-cpp-#{platform.architecture}" else pkg.build_requires "pl-gcc" pkg.build_requires "pl-cmake" - pkg.build_requires "pl-boost" - pkg.build_requires "pl-yaml-cpp" end pkg.add_source("file://resources/files/facter-gem/make.bat") @@ -26,17 +21,17 @@ make = 'make' rm = 'rm' pkg.environment "PATH" => "/usr/local/bin:#{settings[:build_tools_dir]}:#{settings[:ruby_dir]}:$$PATH" - pkg.environment('FACTER_CMAKE_OPTS', "-DBOOST_STATIC=ON -DYAMLCPP_STATIC=ON -DLEATHERMAN_USE_CURL=FALSE -DWITHOUT_CURL=TRUE -DWITHOUT_OPENSSL=TRUE -DWITHOUT_BLKID=TRUE -DFACTER_SKIP_TESTS=TRUE -DWITHOUT_JRUBY=ON") + pkg.environment('FACTER_CMAKE_OPTS', "-DLEATHERMAN_USE_CURL=FALSE -DWITHOUT_CURL=TRUE -DWITHOUT_OPENSSL=TRUE -DWITHOUT_BLKID=TRUE -DFACTER_SKIP_TESTS=TRUE -DWITHOUT_JRUBY=ON") elsif platform.is_windows? make = "#{settings[:gcc_bindir]}/mingw32-make" rm = '/bin/rm' - pkg.environment "PATH" => "/cygdrive/c/ProgramData/chocolatey/bin:$$(cygpath -u #{settings[:gcc_bindir]}):$$(cygpath -u #{settings[:build_tools_dir]}):$$(cygpath -u #{settings[:ruby_dir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" - pkg.environment('FACTER_CMAKE_OPTS', '-G \"MinGW Makefiles\" -DBOOST_STATIC=ON -DYAMLCPP_STATIC=ON -DLEATHERMAN_USE_CURL=FALSE -DWITHOUT_CURL=TRUE -DWITHOUT_OPENSSL=TRUE -DWITHOUT_BLKID=TRUE -DFACTER_SKIP_TESTS=TRUE -DCMAKE_TOOLCHAIN_FILE=C:\tools\pl-build-tools\pl-build-toolchain.cmake -DWITHOUT_JRUBY=ON') + pkg.environment "PATH", "$(shell cygpath -u #{settings[:prefix]}/lib):/cygdrive/c/ProgramData/chocolatey/bin:$(cygpath -u #{settings[:gcc_bindir]}):$(cygpath -u #{settings[:build_tools_dir]}):$(cygpath -u #{settings[:ruby_dir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" + pkg.environment('FACTER_CMAKE_OPTS', '-G \"MinGW Makefiles\" -DLEATHERMAN_USE_CURL=FALSE -DWITHOUT_CURL=TRUE -DWITHOUT_OPENSSL=TRUE -DWITHOUT_BLKID=TRUE -DFACTER_SKIP_TESTS=TRUE -DCMAKE_TOOLCHAIN_FILE=C:\tools\pl-build-tools\pl-build-toolchain.cmake -DWITHOUT_JRUBY=ON') else make = 'make' rm = 'rm' pkg.environment "PATH" => "#{settings[:build_tools_dir]}:#{settings[:ruby_dir]}:$$PATH" - pkg.environment('FACTER_CMAKE_OPTS', "-DBOOST_STATIC=ON -DYAMLCPP_STATIC=ON -DLEATHERMAN_USE_CURL=FALSE -DWITHOUT_CURL=TRUE -DWITHOUT_OPENSSL=TRUE -DWITHOUT_BLKID=TRUE -DFACTER_SKIP_TESTS=TRUE -DWITHOUT_JRUBY=ON") + pkg.environment('FACTER_CMAKE_OPTS', "-DLEATHERMAN_USE_CURL=FALSE -DWITHOUT_CURL=TRUE -DWITHOUT_OPENSSL=TRUE -DWITHOUT_BLKID=TRUE -DFACTER_SKIP_TESTS=TRUE -DWITHOUT_JRUBY=ON") end if platform.is_windows? diff --git a/configs/components/facter.rb b/configs/components/facter.rb index 49e411f207..a632e1ab72 100644 --- a/configs/components/facter.rb +++ b/configs/components/facter.rb @@ -18,7 +18,7 @@ pkg.build_requires 'openssl-devel' end - pkg.build_requires 'puppet-runtime' # Provides openssl, ruby, augeas, curl + pkg.build_requires 'puppet-runtime' # Provides augeas, boost, curl, openssl, ruby, yaml-cpp pkg.build_requires 'leatherman' pkg.build_requires 'runtime' pkg.build_requires 'cpp-hocon' @@ -30,27 +30,11 @@ end if platform.is_windows? - pkg.environment "PATH", "$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:ruby_bindir]}):$(shell cygpath -u #{settings[:bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" + pkg.environment "PATH", "$(shell cygpath -u #{settings[:prefix]}/lib):$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:ruby_bindir]}):$(shell cygpath -u #{settings[:bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" else pkg.environment "PATH", "#{settings[:bindir]}:$(PATH)" end - if platform.is_macos? - pkg.build_requires "yaml-cpp" - elsif platform.name =~ /solaris-10/ - pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/solaris/10/pl-yaml-cpp-0.5.1.#{platform.architecture}.pkg.gz" - elsif platform.is_cross_compiled_linux? || platform.name =~ /solaris-11/ - pkg.build_requires "pl-yaml-cpp-#{platform.architecture}" - elsif platform.is_aix? - pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-gcc-5.2.0-11.aix#{platform.os_version}.ppc.rpm" - pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-cmake-3.2.3-2.aix#{platform.os_version}.ppc.rpm" - pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-yaml-cpp-0.5.1-1.aix#{platform.os_version}.ppc.rpm" - elsif platform.is_windows? - pkg.build_requires "pl-yaml-cpp-#{platform.architecture}" - else - pkg.build_requires "pl-yaml-cpp" - end - # Explicitly skip jruby if not installing a jdk. skip_jruby = 'OFF' java_home = '' @@ -172,8 +156,6 @@ -DCMAKE_PREFIX_PATH=#{settings[:prefix]} \ -DCMAKE_INSTALL_RPATH=#{settings[:libdir]} \ #{special_flags} \ - -DBOOST_STATIC=ON \ - -DYAMLCPP_STATIC=ON \ -DWITHOUT_CURL=#{skip_curl} \ -DWITHOUT_BLKID=#{skip_blkid} \ -DWITHOUT_JRUBY=#{skip_jruby} \ @@ -237,6 +219,15 @@ pkg.install_file "#{settings[:prefix]}/bin/#{dll}", "#{settings[:facter_root]}/bin/#{dll}" end + # Also copy the boost dlls that facter requires into the ruby and facter + # bindirs, so that dynamically linked executables can find them + settings[:boost_libs].each do |name| + pkg.install_file "#{settings[:prefix]}/lib/libboost_#{name}.dll", "#{settings[:facter_root]}/bin/libboost_#{name}.dll" + pkg.install_file "#{settings[:prefix]}/lib/libboost_#{name}.dll.a", "#{settings[:facter_root]}/bin/libboost_#{name}.dll.a" + pkg.install_file "#{settings[:prefix]}/lib/libboost_#{name}.dll", "#{settings[:ruby_bindir]}/libboost_#{name}.dll" + pkg.install_file "#{settings[:prefix]}/lib/libboost_#{name}.dll.a", "#{settings[:ruby_bindir]}/libboost_#{name}.dll.a" + end + # Copy these into both facter and ruby's bindirs [ "leatherman_curl.dll", diff --git a/configs/components/leatherman.rb b/configs/components/leatherman.rb index 0707943e2b..da6c9987d9 100644 --- a/configs/components/leatherman.rb +++ b/configs/components/leatherman.rb @@ -5,31 +5,25 @@ if platform.is_macos? pkg.build_requires "cmake" - pkg.build_requires "boost" pkg.build_requires "gettext" elsif platform.name =~ /solaris-10/ - pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/solaris/10/pl-boost-1.58.0-7.#{platform.architecture}.pkg.gz" pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/solaris/10/pl-cmake-3.2.3-2.i386.pkg.gz" elsif platform.is_cross_compiled_linux? || platform.name =~ /solaris-11/ - pkg.build_requires "pl-boost-#{platform.architecture}" pkg.build_requires "pl-cmake" elsif platform.is_aix? pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-gcc-5.2.0-11.aix#{platform.os_version}.ppc.rpm" pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-cmake-3.2.3-2.aix#{platform.os_version}.ppc.rpm" - pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-boost-1.58.0-7.aix#{platform.os_version}.ppc.rpm" pkg.build_requires "http://pl-build-tools.delivery.puppetlabs.net/aix/#{platform.os_version}/ppc/pl-gettext-0.19.8-2.aix#{platform.os_version}.ppc.rpm" elsif platform.is_windows? pkg.build_requires "cmake" pkg.build_requires "pl-toolchain-#{platform.architecture}" - pkg.build_requires "pl-boost-#{platform.architecture}" pkg.build_requires "pl-gettext-#{platform.architecture}" else pkg.build_requires "pl-cmake" - pkg.build_requires "pl-boost" pkg.build_requires "pl-gettext" end - pkg.build_requires "puppet-runtime" # Provides curl and ruby + pkg.build_requires "puppet-runtime" # Provides boost, curl, ruby pkg.build_requires "runtime" ruby = "#{settings[:host_ruby]} -rrbconfig" @@ -58,7 +52,7 @@ special_flags = "-DCMAKE_CXX_FLAGS_RELEASE='-O2 -DNDEBUG'" elsif platform.is_windows? make = "#{settings[:gcc_bindir]}/mingw32-make" - pkg.environment "PATH", "$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:ruby_bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" + pkg.environment "PATH", "$(shell cygpath -u #{settings[:prefix]}/lib):$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:ruby_bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" pkg.environment "CYGWIN", settings[:cygwin] cmake = "C:/ProgramData/chocolatey/bin/cmake.exe -G \"MinGW Makefiles\"" @@ -91,7 +85,6 @@ #{leatherman_locale_var} \ -DLEATHERMAN_SHARED=TRUE \ #{special_flags} \ - -DBOOST_STATIC=ON \ ."] end diff --git a/configs/components/libwhereami.rb b/configs/components/libwhereami.rb index a41dc37e49..bf8fb2c08e 100644 --- a/configs/components/libwhereami.rb +++ b/configs/components/libwhereami.rb @@ -22,7 +22,7 @@ special_flags = "-DCMAKE_CXX_FLAGS_RELEASE='-O2 -DNDEBUG'" elsif platform.is_windows? make = "#{settings[:gcc_bindir]}/mingw32-make" - pkg.environment "PATH", "$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" + pkg.environment "PATH", "$(shell cygpath -u #{settings[:prefix]}/lib):$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" pkg.environment "CYGWIN", settings[:cygwin] cmake = "C:/ProgramData/chocolatey/bin/cmake.exe -G \"MinGW Makefiles\"" @@ -44,8 +44,8 @@ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_PREFIX_PATH=#{settings[:prefix]} \ -DCMAKE_INSTALL_PREFIX=#{settings[:prefix]} \ + -DCMAKE_INSTALL_RPATH=#{settings[:libdir]} \ #{special_flags} \ - -DBOOST_STATIC=ON \ ."] end diff --git a/configs/components/pxp-agent.rb b/configs/components/pxp-agent.rb index e4abb74de1..cefd93aac3 100644 --- a/configs/components/pxp-agent.rb +++ b/configs/components/pxp-agent.rb @@ -5,7 +5,7 @@ cmake = "/opt/pl-build-tools/bin/cmake" if platform.is_windows? - pkg.environment "PATH", "$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:ruby_bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" + pkg.environment "PATH", "$(shell cygpath -u #{settings[:prefix]}/lib):$(shell cygpath -u #{settings[:gcc_bindir]}):$(shell cygpath -u #{settings[:ruby_bindir]}):/cygdrive/c/Windows/system32:/cygdrive/c/Windows:/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0" else pkg.environment "PATH", "#{settings[:bindir]}:/opt/pl-build-tools/bin:$(PATH)" end @@ -62,7 +62,6 @@ -DCMAKE_SYSTEM_PREFIX_PATH=#{settings[:prefix]} \ -DMODULES_INSTALL_PATH=#{File.join(settings[:install_root], 'pxp-agent', 'modules')} \ #{special_flags} \ - -DBOOST_STATIC=ON \ ." ] end @@ -154,4 +153,29 @@ else fail "need to know where to put #{pkg.get_name} service files" end + + # Unless the settings specify that this is a development build, we remove + # unneeded header files so that they don't make it into the final package + # (boost headers, for example, increase the size of the package to an + # unacceptable degree). + # + # We're doing this in the pxp-agent component because pxp-agent is the last + # component to build that requires these headers. + unless settings[:dev_build] + # Note that ruby is _not_ included in this list because its headers are + # required to build native extensions for gems. + unwanted_headers = ["augeas.h", "boost", "cpp-pcp-client", "curl", "fa.h", + "facter", "hocon", "leatherman", "libexslt", "libxml2", + "libxslt", "openssl", "whereami", "yaml-cpp"] + + # We need a full path on windows because /usr/bin is not in the PATH at this point + rm = platform.is_windows? ? '/usr/bin/rm' : 'rm' + + pkg.install do + [ + unwanted_headers.map { |h| "#{rm} -rf #{settings[:includedir]}/#{h}" }, + "#{rm} -rf #{settings[:prefix]}/ssl/man", # Also remove unwanted OpenSSL manpages + ] + end + end end diff --git a/configs/components/runtime.rb b/configs/components/runtime.rb index 8016e7dca6..f804abb916 100644 --- a/configs/components/runtime.rb +++ b/configs/components/runtime.rb @@ -58,7 +58,7 @@ # update curl to statically link against zlib pkg.install_file "#{settings[:tools_root]}/bin/zlib1.dll", "#{settings[:ruby_bindir]}/zlib1.dll" - # gdbm, yaml-cpp and iconv are all runtime dependancies of ruby, and their libraries need + # gdbm and iconv are runtime dependancies of ruby, and their libraries need # To exist inside our vendored ruby pkg.install_file "#{settings[:tools_root]}/bin/libgdbm-4.dll", "#{settings[:ruby_bindir]}/libgdbm-4.dll" pkg.install_file "#{settings[:tools_root]}/bin/libgdbm_compat-4.dll", "#{settings[:ruby_bindir]}/libgdbm_compat-4.dll"