Merge branch 'master' into loki

crowdsecurity · Jun 28, 2023 · 4f46ea4 · 4f46ea4
2 parents d6988c2 + ebe25d7
commit 4f46ea4
Show file tree

Hide file tree

Showing 34 changed files with 714 additions and 460 deletions.
diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml
@@ -27,37 +27,23 @@ jobs:
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-
     - name: "Check out CrowdSec repository"
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: Cache Go modules
-      uses: actions/cache@v3
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v4
       with:
-        path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-          ~/Library/Caches/go-build
-          %LocalAppData%\go-build
-        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.go-version }}-go-
+        go-version: ${{ matrix.go-version }}
+        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
         sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
-        go install github.com/mikefarah/yq/v4@latest
-        go install github.com/cloudflare/cfssl/cmd/cfssl@master
-        go install github.com/cloudflare/cfssl/cmd/cfssljson@master
 
     - name: "Build crowdsec and fixture"
       run: make bats-clean bats-build bats-fixture BUILD_STATIC=1

diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml
@@ -34,37 +34,23 @@ jobs:
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-
     - name: "Check out CrowdSec repository"
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: Cache Go modules
-      uses: actions/cache@v3
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v4
       with:
-        path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-          ~/Library/Caches/go-build
-          %LocalAppData%\go-build
-        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.go-version }}-go-
+        go-version: ${{ matrix.go-version }}
+        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
         sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
-        go install github.com/mikefarah/yq/v4@latest
-        go install github.com/cloudflare/cfssl/cmd/cfssl@master
-        go install github.com/cloudflare/cfssl/cmd/cfssljson@master
 
     - name: "Build crowdsec and fixture"
       run: |

diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml
@@ -35,37 +35,23 @@ jobs:
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-
     - name: "Check out CrowdSec repository"
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: Cache Go modules
-      uses: actions/cache@v3
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v4
       with:
-        path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-          ~/Library/Caches/go-build
-          %LocalAppData%\go-build
-        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.go-version }}-go-
+        go-version: ${{ matrix.go-version }}
+        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
         sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
-        go install github.com/mikefarah/yq/v4@latest
-        go install github.com/cloudflare/cfssl/cmd/cfssl@master
-        go install github.com/cloudflare/cfssl/cmd/cfssljson@master
 
     - name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
       run: |

diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml
@@ -24,37 +24,23 @@ jobs:
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-
     - name: "Check out CrowdSec repository"
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: Cache Go modules
-      uses: actions/cache@v3
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v4
       with:
-        path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-          ~/Library/Caches/go-build
-          %LocalAppData%\go-build
-        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.go-version }}-go-
+        go-version: ${{ matrix.go-version }}
+        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
         sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
-        go install github.com/mikefarah/yq/v4@latest
-        go install github.com/cloudflare/cfssl/cmd/cfssl@master
-        go install github.com/cloudflare/cfssl/cmd/cfssljson@master
 
     - name: "Build crowdsec and fixture"
       run: |

diff --git a/.github/workflows/ci-windows-build-msi.yml b/.github/workflows/ci-windows-build-msi.yml
@@ -30,28 +30,17 @@ jobs:
 
     steps:
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: false
 
-    - name: Cache Go modules
-      uses: actions/cache@v3
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v4
       with:
-        path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-          ~/Library/Caches/go-build
-          %LocalAppData%\go-build
-        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.go-version }}-go-
+        go-version: ${{ matrix.go-version }}
+        cache-dependency-path: "**/go.sum"
 
     - name: Build
       run: make windows_installer BUILD_RE2_WASM=1

diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml
@@ -30,18 +30,6 @@ jobs:
         with:
           config: .github/buildkit.toml
 
-      # - name: "Build flavor: full"
-      #   uses: docker/build-push-action@v4
-      #   with:
-      #     context: .
-      #     file: ./Dockerfile
-      #     tags: crowdsecurity/crowdsec:test
-      #     target: full
-      #     platforms: linux/amd64
-      #     load: true
-      #     cache-from: type=gha
-      #     cache-to: type=gha,mode=min
-
       - name: "Build flavor: slim"
         uses: docker/build-push-action@v4
         with:

diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml
@@ -29,28 +29,17 @@ jobs:
 
     steps:
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-
     - name: Check out CrowdSec repository
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: false
 
-    - name: Cache Go modules
-      uses: actions/cache@v3
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v4
       with:
-        path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-          ~/Library/Caches/go-build
-          %LocalAppData%\go-build
-        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.go-version }}-go-
+        go-version: ${{ matrix.go-version }}
+        cache-dependency-path: "**/go.sum"
 
     - name: Build
       run: |

diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml
@@ -110,28 +110,17 @@ jobs:
 
     steps:
 
-    - name: "Set up Go ${{ matrix.go-version }}"
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-
     - name: Check out CrowdSec repository
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: false
 
-    - name: Cache Go modules
-      uses: actions/cache@v3
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v4
       with:
-        path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-          ~/Library/Caches/go-build
-          %LocalAppData%\go-build
-        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.go-version }}-go-
+        go-version: ${{ matrix.go-version }}
+        cache-dependency-path: "**/go.sum"
 
     - name: Build and run tests, static
       run: |

diff --git a/.github/workflows/release_publish-package.yml b/.github/workflows/release_publish-package.yml
@@ -20,28 +20,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-      - name: "Set up Go ${{ matrix.go-version }}"
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ matrix.go-version }}
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
           submodules: false
 
-      - name: Cache Go modules
-        uses: actions/cache@v3
+      - name: "Set up Go ${{ matrix.go-version }}"
+        uses: actions/setup-go@v4
         with:
-          path: |
-            ~/go/pkg/mod
-            ~/.cache/go-build
-            ~/Library/Caches/go-build
-            %LocalAppData%\go-build
-          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.go-version }}-go-
+          go-version: ${{ matrix.go-version }}
+          cache-dependency-path: "**/go.sum"
 
       - name: Build the binaries
         run: |

diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,9 @@
 *.test
 *.cover
 
+# Test dependencies
+test/tools/*
+
 # VMs used for dev/test
 
 .vagrant

diff --git a/Dockerfile b/Dockerfile
@@ -5,29 +5,28 @@ FROM golang:${GOVERSION}-alpine AS build
 
 WORKDIR /go/src/crowdsec
 
-COPY . .
-
-# Alpine does not ship a static version of re2, we can build it ourselves
-# Later versions require 'abseil', which is likewise not available in its static form
+# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway.
 ENV RE2_VERSION=2023-03-01
 
 # wizard.sh requires GNU coreutils
-RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \
+RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static pkgconfig && \
     wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
     tar -xzf ${RE2_VERSION}.tar.gz && \
     cd re2-${RE2_VERSION} && \
     make && \
     make install && \
     echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
-    cd - && \
-    make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
+    go install github.com/mikefarah/yq/v4@v4.34.1
+
+COPY . .
+
+RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     cd crowdsec-v* && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
     cscli collections install crowdsecurity/linux && \
-    cscli parsers install crowdsecurity/whitelists && \
-    go install github.com/mikefarah/yq/v4@v4.31.2
+    cscli parsers install crowdsecurity/whitelists
 
     # In case we need to remove agents here..
     # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete