Merge branch 'main' into evgb-ProcessTestsInitial

.github/ISSUE_TEMPLATE/csp_task.md

@@ -0,0 +1,40 @@
+---
+name: Cloud Security Posture Task
+about: Create an issue for the Cloud Security Posture Team.
+title: ""
+labels: "Team: csp"
+assignees: ''
+
+---
+
+
+**Motivation**
+
+Why are we doing this task? what it the value to the user. 
+
+**Definition of done**
+
+What needs to be completed at the end of this task 
+
+**Out of scope**
+
+What is not included in this task  
+
+
+**Related tasks/epics**
+
+Reference related issues and epics
+
+**Checklist**
+
+Please follow the following checklist in the beginning of your work, please comment with a suggested of high level solution. It should include:
+- [ ] Comment describing high level implementation details
+   - [ ] Include API and data models
+   - [ ] Include assumptions being taken
+   - [ ] Provide backward/forward compatibility when changing data model schemas and key constants
+   - [ ] Mention relevant individuals with a reason (getting feedback, fyi etc)
+- [ ] Submit a PR for our [technical index](https://github.com/elastic/security-team/blob/main/docs/cloud-security-posture-team/Technical_Index.md) that includes breaking changes/ new features
+
+**Before closing this ticket**
+- [ ] Commit the [technical index](https://github.com/elastic/security-team/blob/main/docs/cloud-security-posture-team/Technical_Index.md) PR
+- [ ] Reference to tech-debts that shall be solved as we move forward

.github/workflows/cloudbeat-ci.yml

@@ -1,12 +1,11 @@
 name: Cloudbeat-CI
 
 on:
-  pull_request_target:
+  pull_request:
     branches:
       - main
     types: [assigned, opened, synchronize, reopened]
 
-
 env:
   K8S_MANIFEST_DIR: deploy/k8s/kustomize/base
   GO_VERSION: 1.17.2
@@ -19,10 +18,6 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          repository: ${{ (github.event.pull_request_target || github.event.pull_request).head.repo.full_name }}
-          ref: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }}
 
       - name: Set up Go
         uses: actions/setup-go@v2
@@ -93,10 +88,6 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          repository: ${{ (github.event.pull_request_target || github.event.pull_request).head.repo.full_name }}
-          ref: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }}
 
       - uses: azure/setup-helm@v1
         id: install
@@ -155,68 +146,3 @@ jobs:
         if: steps.deploy_helm.outcome == 'failure'
         with:
           limit-access-to-actor: true
-
-      - name: Get Allure history
-        uses: actions/checkout@v2
-        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
-        continue-on-error: true
-        with:
-          ref: gh-pages
-          path: gh-pages
-
-      - name: Generate
-        uses: simple-elf/allure-report-action@master
-        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
-        with:
-          allure_results: tests/allure/results
-          allure_history: allure-history
-          keep_reports: 20
-
-      - name: Deploy report to Github Pages
-        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
-        uses: peaceiris/actions-gh-pages@v3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_branch: gh-pages
-          publish_dir: allure-history
-
-      - name: Comment test success result
-        uses: marocchino/sticky-pull-request-comment@v2
-        if: steps.run_tests.outcome == 'success'
-        with:
-          header: Test Results
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          message: |
-            ## Cloudbeat CI :robot:
-            Integration tests passed! :green_heart:
-            Tests Summary:
-            ```python
-            ${{env.summary}}
-            ```
-            Link to detailed report: https://${{ github.repository_owner }}.github.io/cloudbeat/${{ github.run_number }}
-
-      - name: Comment test failed result
-        uses: marocchino/sticky-pull-request-comment@v2
-        if: steps.run_tests.outcome == 'failure'
-        with:
-          header: Test Results
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          message: |
-            ## Cloudbeat CI :robot:
-            Integration tests failed! :broken_heart:
-            Tests Summary:
-            ```python
-            ${{env.summary}}
-            ```
-            Link to detailed report: https://${{ github.repository_owner }}.github.io/cloudbeat/${{ github.run_number }}
-
-      - name: Comment test skipped result
-        uses: marocchino/sticky-pull-request-comment@v2
-        if: (steps.run_tests.outcome == 'skipped' || steps.run_tests.outcome == 'cancelled')
-        with:
-          header: Test Results
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          message: |
-            ## Cloudbeat CI :robot:
-            Integration tests status: ${{ steps.run_tests.outcome }}
-

.github/workflows/unit-test.yml

@@ -21,10 +21,6 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          repository: ${{ (github.event.pull_request_target || github.event.pull_request).head.repo.full_name }}
-          ref: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }}
 
       - name: Set up Go
         uses: actions/setup-go@v2
@@ -59,10 +55,6 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          repository: ${{ (github.event.pull_request_target || github.event.pull_request).head.repo.full_name }}
-          ref: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }}
 
       - name: Get diff k8s manifests
         uses: technote-space/get-diff-action@v4

_meta/config/cloudbeat.common.yml.tmpl

@@ -1,6 +1,6 @@
 cloudbeat:
   # Defines how often an event is sent to the output
-  period: 30s
+  period: 4h
   fetchers:
   - name: kube-api
   - name: process

cloudbeat.reference.yml

@@ -9,7 +9,7 @@
 
 cloudbeat:
   # Defines how often an event is sent to the output
-  period: 30s
+  period: 4h
   fetchers:
   - name: kube-api
   - name: process

cloudbeat.yml

@@ -12,7 +12,7 @@
 
 cloudbeat:
   # Defines how often an event is sent to the output
-  period: 30s
+  period: 4h
   fetchers:
   - name: kube-api
   - name: process

config/config.go

@@ -50,7 +50,7 @@ type Stream struct {
 }
 
 var DefaultConfig = Config{
-	Period: 10 * time.Second,
+	Period: 4 * time.Hour,
 }
 
 func New(cfg *common.Config) (Config, error) {

config/config_test.go

@@ -300,3 +300,25 @@ activated_rules:
 		s.Equal(strings.TrimSpace(test.expected), strings.TrimSpace(dy))
 	}
 }
+
+func (s *ConfigTestSuite) TestConfigPeriod() {
+	var tests = []struct {
+		config         string
+		expectedPeriod time.Duration
+	}{
+		{"", 4 * time.Hour},
+		{"period: 50s", 50 * time.Second},
+		{"period: 5m", 5 * time.Minute},
+		{"period: 2h", 2 * time.Hour},
+	}
+
+	for _, test := range tests {
+		cfg, err := common.NewConfigFrom(test.config)
+		s.NoError(err)
+
+		c, err := New(cfg)
+		s.NoError(err)
+
+		s.Equal(test.expectedPeriod, c.Period)
+	}
+}

deploy/eks/cloudbeat-ds.yml

@@ -49,6 +49,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            # This is done for leader election on local env
+            # GAP: https://github.com/elastic/cloudbeat/issues/162
+            - name: POD_NAME
+              value: cloudbeat-leader
+                # fieldRef:
+                #   fieldPath: metadata.name
           securityContext:
             runAsUser: 0
             # If using Red Hat OpenShift uncomment this:
@@ -330,3 +336,14 @@ roleRef:
   name: cloudbeat-kubeadm-config
   apiGroup: rbac.authorization.k8s.io
 ---
+# GAP: https://github.com/elastic/cloudbeat/issues/162
+apiVersion: coordination.k8s.io/v1
+kind: Lease
+metadata:
+  name: elastic-agent-cluster-leader
+  namespace: kube-system
+spec:
+  holderIdentity: cloudbeat-leader
+  leaseDurationSeconds: 15
+  renewTime: "1992-04-26T00:00:00.000000Z"
+---

deploy/k8s/cloudbeat-ds-debug.yml

@@ -46,6 +46,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            # This is done for leader election on local env
+            # GAP: https://github.com/elastic/cloudbeat/issues/162
+            - name: POD_NAME
+              value: cloudbeat-leader
+                # fieldRef:
+                #   fieldPath: metadata.name
           securityContext:
             runAsUser: 0
             # If using Red Hat OpenShift uncomment this:
@@ -359,4 +365,14 @@ roleRef:
   name: cloudbeat-kubeadm-config
   apiGroup: rbac.authorization.k8s.io
 ---
-
+# GAP: https://github.com/elastic/cloudbeat/issues/162
+apiVersion: coordination.k8s.io/v1
+kind: Lease
+metadata:
+  name: elastic-agent-cluster-leader
+  namespace: kube-system
+spec:
+  holderIdentity: cloudbeat-leader
+  leaseDurationSeconds: 15
+  renewTime: "1992-04-26T00:00:00.000000Z"
+---

deploy/k8s/kustomize/base/cloudbeat-ds.yml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: cloudbeat
-          image: cloudbeat:latest 
+          image: cloudbeat:latest
           imagePullPolicy: IfNotPresent
           env:
             - name: ES_HOST
@@ -37,6 +37,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            # This is done for leader election on local env
+            # GAP: https://github.com/elastic/cloudbeat/issues/162
+            - name: POD_NAME
+              value: cloudbeat-leader
+                # fieldRef:
+                #   fieldPath: metadata.name
           securityContext:
             runAsUser: 0
             # If using Red Hat OpenShift uncomment this:
@@ -188,7 +194,7 @@ data:
       #api_key: "id:api_key"
       username: ${ES_USERNAME}
       password: ${ES_PASSWORD}
-    
+
       # Enable to allow sending output to older ES versions
       allow_older_versions: true
 
@@ -241,11 +247,11 @@ rules:
   #  resources:
   #  - secrets
   #  verbs: ["get"]
-  - apiGroups: [ "extensions" ]
+  - apiGroups: ["extensions"]
     resources:
       - replicasets
-    verbs: [ "get", "list", "watch" ]
-  - apiGroups: [ "apps" ]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
     resources:
       - statefulsets
       - deployments
@@ -353,4 +359,15 @@ roleRef:
   kind: Role
   name: cloudbeat-kubeadm-config
   apiGroup: rbac.authorization.k8s.io
----
+---
+# GAP: https://github.com/elastic/cloudbeat/issues/162
+apiVersion: coordination.k8s.io/v1
+kind: Lease
+metadata:
+  name: elastic-agent-cluster-leader
+  namespace: kube-system
+spec:
+  holderIdentity: cloudbeat-leader
+  leaseDurationSeconds: 15
+  renewTime: "1992-04-26T00:00:00.000000Z"
+---

resources/conditions/kube_lease.go

@@ -49,6 +49,10 @@ func (l *leaseProvider) IsLeader() (bool, error) {
 		return DefaultLeaderValue, fmt.Errorf("could not find lease %v: %s", DefaultLeaderLeaseName, err.Error())
 	}
 
+	if (lease.Spec.HolderIdentity == nil) {
+		return DefaultLeaderValue, fmt.Errorf("lease %v HolderIdentity is empty", DefaultLeaderLeaseName)
+	}
+
 	podId := lastPart(*lease.Spec.HolderIdentity)
 
 	return podId == l.currentPodID(), nil

resources/conditions/kube_lease_test.go

@@ -123,3 +123,28 @@ func TestKubeLeaseNoLeader(t *testing.T) {
 	assert.Error(t, err)
 	assert.False(t, result)
 }
+
+func TestKubeLeaseEmptyLeader(t *testing.T) {
+	t.Setenv("POD_NAME", "my_cloudbeat")
+
+	leases := v1.LeaseList{Items: []v1.Lease{
+		{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Lease",
+				APIVersion: "v1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "elastic-agent-cluster-leader",
+				Namespace: "kube-system",
+			},
+			Spec: v1.LeaseSpec{},
+		},
+	}}
+
+	client := k8sfake.NewSimpleClientset(&leases)
+	provider := NewLeaderLeaseProvider(context.TODO(), client)
+
+	result, err := provider.IsLeader()
+	assert.NotNil(t, err, "IsLeader disregards empty holder")
+	assert.False(t, result)
+}