First steps to remove the Buck build system (#6361)

* First steps to remove the Buck build system * Azure Pipelines: Rename jobs for consistency
osquery · Apr 8, 2020 · 52f310a · 52f310a
1 parent 8937442
commit 52f310a
Show file tree

Hide file tree

Showing 233 changed files with 23 additions and 12,732 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -25,7 +25,6 @@ Here are some common things we look for:
 - The minimal amount of includes are used, only include what you use.
 - Comments for methods, structures, and classes follow our common patterns.
 - `Status` and `LOG(N)` messages do not use punctuation or contractions.
-- Support for both CMake and BUCK (we are happy to help).
 - The code mostly looks and feels similar to the existing codebase.
 
 -->
diff --git a/.gitignore b/.gitignore
@@ -70,10 +70,3 @@ cscope.out
 /external/*
 !/external/cmake/cmakelibs.cmake
 !/external/CMakeLists.txt
-
-# Buck
-.buckd/
-buck-out/
-
-# Windows system specific toolchain paths
-tools/buckconfigs/windows-x86_64/toolchain/vs2017_15.5.bcfg
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -5,7 +5,8 @@ jobs:
 
 # LINUX
 
-  - job: LinuxCMake
+  - job: LinuxBuild
+    displayName: "Linux"
 
     strategy:
       matrix:
@@ -118,40 +119,6 @@ jobs:
       displayName: "Reclaim disk space"
 
 
-  - job: LinuxBuck
-    displayName: "LinuxBuck Release"
-
-    pool:
-      vmImage: 'Ubuntu-16.04'
-
-    container:
-      image: trailofbits/osql-experimental:ubuntu-18.04
-      options: --privileged
-
-    steps:
-    - script: |
-        wget https://github.com/facebook/buck/releases/download/v2019.06.17.01/buck.2019.06.17.01_all.deb
-        sudo apt update
-        sudo apt install -q -y --no-install-recommends ./buck.2019.06.17.01_all.deb openjdk-8-jre python3-distutils
-      workingDirectory: $(Build.BinariesDirectory)
-
-    - script: |
-        export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
-        buck build @mode/linux-x86_64/release osquery:osqueryd
-      displayName: "Build osquery"
-
-    - script: |
-        # Buck compiles these tests in Debug mode, exhausting disk space.
-        # Until we find a solution, we'll disable the tests.
-        #export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
-        #buck test @mode/linux-x86_64/release osquery/... tests/... plugins/...
-      displayName: "Run tests (DISABLED)"
-
-    - script: |
-        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
-      name: JobResult
-
-
   - job: Linux
 
     pool:
@@ -160,21 +127,17 @@ jobs:
     condition: succeededOrFailed()
 
     dependsOn:
-      - LinuxCMake
-      - LinuxBuck
+      - LinuxBuild
 
     variables:
-      LinuxCMakeReleaseStatus: $[ dependencies.LinuxCMake.outputs['Release.JobResult.Status'] ]
-      LinuxCMakeDebugStatus: $[ dependencies.LinuxCMake.outputs['Debug.JobResult.Status'] ]
-      LinuxBuckReleaseStatus: $[ dependencies.LinuxBuck.outputs['JobResult.Status'] ]
+      LinuxReleaseStatus: $[ dependencies.LinuxBuild.outputs['Release.JobResult.Status'] ]
+      LinuxDebugStatus: $[ dependencies.LinuxBuild.outputs['Debug.JobResult.Status'] ]
 
     steps:
     - checkout: none
 
     - script: |
-        if [ -z "$(LinuxCMakeReleaseStatus)" ] || [ -z "$(LinuxCMakeDebugStatus)" ] || \
-           [ -z "$(LinuxBuckReleaseStatus)" ]
-        then
+        if [ -z "$(LinuxReleaseStatus)" ] || [ -z "$(LinuxDebugStatus)" ]; then
           exit 1
         fi
       displayName: "Detect Linux jobs build statuses"
@@ -183,7 +146,8 @@ jobs:
 
 # MACOS
 
-  - job: macOSCMake
+  - job: macOSBuild
+    displayName: "macOS"
 
     strategy:
       matrix:
@@ -267,38 +231,6 @@ jobs:
       displayName: "Reclaim disk space"
 
 
-  - job: macOSBuck
-    displayName: "macOSBuck Release"
-
-    pool:
-      vmImage: macos-10.14
-
-    steps:
-    - script: |
-        brew update
-        brew tap facebook/fb
-        brew upgrade
-        brew cask install adoptopenjdk
-        brew install buck watchman
-        sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer
-      displayName: "Install Homebrew and prerequisites"
-      timeoutInMinutes: 20
-
-    - script: |
-        export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
-        buck build @mode/macos-x86_64/release osquery:osqueryd
-      displayName: "Build osquery"
-
-    - script: |
-        export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
-        buck test @mode/macos-x86_64/release osquery/... tests/... plugins/...
-      displayName: "Run tests"
-
-    - script: |
-        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
-      name: JobResult
-
-
   - job: macOS
 
     pool:
@@ -307,21 +239,17 @@ jobs:
     condition: succeededOrFailed()
 
     dependsOn:
-      - macOSCMake
-      - macOSBuck
+      - macOSBuild
 
     variables:
-      macOSCMakeReleaseStatus: $[ dependencies.macOSCMake.outputs['Release.JobResult.Status'] ]
-      macOSCMakeDebugStatus: $[ dependencies.macOSCMake.outputs['Debug.JobResult.Status'] ]
-      macOSBuckReleaseStatus: $[ dependencies.macOSBuck.outputs['JobResult.Status'] ]
+      macOSReleaseStatus: $[ dependencies.macOSBuild.outputs['Release.JobResult.Status'] ]
+      macOSDebugStatus: $[ dependencies.macOSBuild.outputs['Debug.JobResult.Status'] ]
 
     steps:
     - checkout: none
 
     - script: |
-        if [ -z "$(macOSCMakeReleaseStatus)" ] || [ -z "$(macOSCMakeDebugStatus)" ] || \
-           [ -z "$(macOSBuckReleaseStatus)" ]
-        then
+        if [ -z "$(macOSReleaseStatus)" ] || [ -z "$(macOSDebugStatus)" ]; then
           exit 1
         fi
       displayName: "Detect macOS jobs build statuses"
@@ -330,8 +258,8 @@ jobs:
 
 # WINDOWS
 
-  - job: WindowsCMake
-    displayName: "WindowsCMake Release"
+  - job: WindowsBuild
+    displayName: "Windows Release"
 
     pool:
       vmImage: vs2017-win2016
@@ -442,64 +370,6 @@ jobs:
         rm -r -Force $(Build.BinariesDirectory)/build
       displayName: "Reclaim disk space"
 
-
-  - job: WindowsBuck
-    displayName: "WindowsBuck Release"
-
-    pool:
-      vmImage: vs2017-win2016
-
-    steps:
-    - powershell: |
-        git config --global core.autocrlf false
-
-    - checkout: self
-
-    - powershell: |
-        choco uninstall mingw -y
-        (New-Object System.Net.WebClient).DownloadFile("https://github.com/facebook/buck/releases/download/v2019.06.17.01/buck.2019.06.17.01.nupkg", `
-        "buck.2019.06.17.01.nupkg")
-        Install-Package buck -Source . -SkipDependencies
-      displayName: "Prepare Buck environment"
-      workingDirectory: $(Build.BinariesDirectory)
-
-    - powershell: |
-        $python2_path = ((Get-Item C:\hostedtoolcache\windows\Python\2*\x64) | Sort-Object -Descending)[0].FullName
-        $python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\x64) | Sort-Object -Descending)[0].FullName
-
-        echo "##vso[task.setvariable variable=python2]$python2_path"
-        echo "##vso[task.setvariable variable=python3]$python3_path"
-
-      displayName: "Detect python 2 and 3 installation folders"
-
-    - script: |
-        mklink /J $(Build.BinariesDirectory)\vsinstall "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise"
-
-    - powershell: |
-        $vcvers = cat $(Build.BinariesDirectory)\vsinstall\VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt
-        .\tools\generate_buck_config.ps1 -VsInstall '$(Build.BinariesDirectory)\vsinstall' `
-        -VcToolsVersion "$vcvers" `
-        -SdkInstall '' `
-        -SdkVersion '' `
-        -Python3Path '$(python3)\python.exe' `
-        -BuckConfigRoot .\tools\buckconfigs\
-      displayName: "Generate Buck config"
-
-    - script: |
-        set PATH=$(python2);%PATH%
-        $(python2)\python.exe "C:\Program Files\PackageManagement\NuGet\Packages\buck.2019.06.17.01\tools\buck.pex" build @mode/windows-x86_64/release osquery:osqueryd
-      displayName: "Build osquery"
-
-    - script: |
-        set PATH=$(python2);%PATH%
-        $(python2)\python.exe "C:\Program Files\PackageManagement\NuGet\Packages\buck.2019.06.17.01\tools\buck.pex" test @mode/windows-x86_64/release osquery/... tests/... plugins/...
-      displayName: "Run tests"
-
-    - powershell: |
-        echo "##vso[task.setvariable variable=Status;isOutput=true]1"
-      name: JobResult
-
-
   - job: Windows
 
     pool:
@@ -508,20 +378,17 @@ jobs:
     condition: succeededOrFailed()
 
     dependsOn:
-      - WindowsCMake
-      - WindowsBuck
+      - WindowsBuild
 
     variables:
-      WindowsCMakeReleaseStatus: $[ dependencies.WindowsCMake.outputs['JobResult.Status'] ]
-      WindowsBuckReleaseStatus: $[ dependencies.WindowsBuck.outputs['JobResult.Status'] ]
+      WindowsReleaseStatus: $[ dependencies.WindowsBuild.outputs['JobResult.Status'] ]
 
     steps:
     - checkout: none
 
     - script: |
-        if [ -z "$(WindowsCMakeReleaseStatus)" ] || [ -z "$(WindowsBuckReleaseStatus)" ]; then
+        if [ -z "$(WindowsReleaseStatus)" ]; then
           exit 1
         fi
-      displayName: "Detect Windows CMake and Buck build status"
-
+      displayName: "Detect Windows build status"
 # WINDOWS
diff --git a/docs/wiki/development/building.md b/docs/wiki/development/building.md
@@ -1,10 +1,10 @@
-osquery supports many flavors of Linux, FreeBSD, macOS, and Windows.
+osquery supports many flavors of Linux, macOS, and Windows.
 
 While osquery runs on a large number of operating systems, we only provide build instructions for a select few.
 
-The supported compilers are: the osquery toolchain (LLVM/Clang 8.0.1) on Linux, MSVC v141 on Windows, and AppleClang from Xcode Command Line Tools 10.2.1.
+The supported compilers are: the osquery toolchain (LLVM/Clang 9.0.1) on Linux, MSVC v141 on Windows, and AppleClang from Xcode Command Line Tools 10.2.1.
 
-# Building with CMake
+# Prerequisites
 
 Git (>= 2.14.0), CMake (>= 3.14.6), Python 3 are required to build. The rest of the dependencies are downloaded by CMake.
 
@@ -230,112 +230,6 @@ By default, the following checks are enabled:
 6. modernize-*
 7. bugprone-*
 
-# Building with Buck
-
-Building and testing is the same on all platforms. Each platform section below describes how to install the required tools and dependencies.
-
-## Linux (Buck)
-
-Install required tools on Ubuntu 18.04 or Ubuntu 18.10:
-
-```bash
-sudo apt install openjdk-8-jre clang libc++1 libc++-dev libc++abi1 libc++abi-dev python python3 python3-distutils
-```
-
-Install library dependencies:
-
-```bash
-sudo apt install liblzma-dev
-```
-
-Install `buck`:
-
-```bash
-wget 'https://github.com/facebook/buck/releases/download/v2018.10.29.01/buck.2018.10.29.01_all.deb'
-sudo apt install ./buck.2018.10.29.01_all.deb
-```
-
-## macOS (Buck)
-
-Install required tools using Homebrew:
-
-```bash
-xcode-select --install
-
-brew tap caskroom/cask
-brew tap caskroom/versions
-brew cask install java8
-```
-
-Install `buck` and `watchman`. Watchman isn't mandatory, but will make builds faster.
-
-```bash
-brew tap facebook/fb
-brew install buck watchman
-```
-
-## FreeBSD (Buck)
-
-Install required tools on FreeBSD 11.2:
-
-```bash
-sudo pkg install openjdk8 python3 python2 clang35
-```
-
-Install `buck`:
-
-```bash
-sudo curl --output /usr/local/bin/buck 'https://jitpack.io/com/github/facebook/buck/v2018.10.29.01/buck-v2018.10.29.01.pex'
-sudo chmod +x /usr/local/bin/buck
-```
-
-Install library dependencies:
-
-```bash
-sudo pkg install glog thrift thrift-cpp boost-libs magic rocksdb-lite rapidjson zstd linenoise-ng augeas ssdeep sleuthkit yara aws-sdk-cpp lldpd libxml++-2 smartmontools lldpd
-```
-
-## Windows 10 (Buck)
-
-You'll need to have the following software installed before you can build osquery on Windows:
-
-- Buck, this also requires the JRE 8 version
-- Visual Studio 2017 or greater
-- The Windows 10 SDK
-- Python3
-
-Once you've installed the above requirements, run `.\tools\generate_buck_config.ps1 -VsInstall '' -VcToolsVersion '' -SdkInstall '' -SdkVersion '' -Python3Path '' -BuckConfigRoot .\tools\buckconfigs\` to generate the buckconfig for building.
-
-## Building and Testing
-
-To build simply run the following command replacing `<platform>` and `<mode>`
-appropriately:
-
-```bash
-buck build @mode/<platform>/<mode> //osquery:osqueryd
-```
-
-When buck finishes find the binary at `buck-out/<mode>/gen/osquery/osqueryd`.
-
-Similarly to run tests just run:
-
-```bash
-buck test @mode/<platform>/<mode> //...
-```
-
-This will run all tests, you can replace `//...` with a specific target to run specific tests only.
-
-Supported platforms:
-
-- `linux-x86_64`
-- `macos-x86_64`
-- `windows-x86_64`
-- `freebsd-x86_64`
-
-Supported modes:
-
-- `release`
-- `debug`
 
 # Using Vagrant
 

diff --git a/libraries/buck/todo b/libraries/buck/todo