Merge branch 'release-3.8.4' into 'develop'

Update version to 3.84 and significant updates to the 'new' event processing... See merge request carbonblack/integrations/event-forwarder!40
carbonblack · Oct 6, 2022 · c7bfa8c · c7bfa8c
2 parents 7e64ef0 + 39afc1f
commit c7bfa8c
Show file tree

Hide file tree

Showing 26 changed files with 260 additions and 135 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,13 @@
 # CB EDR Event Forwarder Changelog
 
+## v3.8.4
+
+#### Bug Fixes / Changes
+
+* The service correctly determines routable URLs for Hosted EDR alert emails.
+* The service correctly processes bundled, compressed event data.
+
+
 ## v3.8.3
 
 #### Bug Fixes / Changes

diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
-GIT_VERSION := 3.8.3
-VERSION := 3.8.3
+GIT_VERSION := 3.8.4
+VERSION := 3.8.4
 GO_PREFIX := github.com/carbonblack/cb-event-forwarder
 EL_VERSION := $(shell rpm -E %{rhel})
 TARGET_OS=linux
@@ -71,17 +71,16 @@ unittest: compile-protobufs
 	go test ./cmd/cb-event-forwarder
 
 test: unittest
-	rm -rf test_output/gold_output
-	rm -rf test_output/go_output
-	rm -rf test_output/leef_output
-	mkdir test_output/gold_output
+	rm -rf test_output
+	rm -rf test_output_old
+	mkdir -p test_output/gold_output
 	python test/scripts/process_events_python.py test/raw_data test_output/gold_output
 	PYTHONIOENCODING=utf8 python test/scripts/compare_outputs.py test_output/gold_output test_output/go_output > test_output/output.txt
 
 clean:
 	rm -f cb-event-forwarder
-	rm -rf test_output/gold_output
-	rm -rf test_output/go_output
+	rm -rf test_output
+	rm -rf test_output_old
 	rm -rf dist
 	rm -rf build
 	rm -f VERSION

diff --git a/README.md b/README.md
@@ -69,10 +69,10 @@ A new dockerized edition of Event Forwarder is now available as of EF 3.8.2 for
 It can be installed with this procedure:
 
 #### Procedure
-1. Retrieve the containerized version of Event Forwarder 3.8.2 with docker using this command:  
-`docker pull projects.registry.vmware.com/carbonblack/event-forwarder:3.8.2`
+1. Retrieve the containerized version of Event Forwarder 3.8.4 with docker using this command:  
+`docker pull projects.registry.vmware.com/carbonblack/event-forwarder:3.8.4`
 2. Retag the downloaded Event Forwarder image using this command:  
-`docker tag projects.registry.vmware.com/carbonblack/event-forwarder:3.8.2 projects.registry.vmware.com/carbonblack/event-forwarder:latest`
+`docker tag projects.registry.vmware.com/carbonblack/event-forwarder:3.8.4 projects.registry.vmware.com/carbonblack/event-forwarder:latest`
 3. From the directory where the edr-docker script is installed, extract the yml file using this command:    
 `docker run --rm --entrypoint=/bin/cat projects.registry.vmware.com/carbonblack/event-forwarder:latest /compose.yml > event-forwarder.yml`
 4. Set up Carbon Black EDR to control Event Forwarder. Edit data/config/cb.conf and add the following values:  
@@ -279,4 +279,4 @@ To build an RPM package, use `make rpm`. Make sure to set the `RPM_OUTPUT_DIR` e
 
 ## Changelog
 
-See CHANGELOG.md.
+See CHANGELOG.md.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -74,9 +74,38 @@ val jsonModelGenerationTask = tasks.register<Exec>("runEasyJson") {
     args("generateeasyjsonmodels")
 }
 
+val buildEventForwarderTask = tasks.register<Exec>("buildEventForwarder") {
+    dependsOn(depTask)
+    dependsOn(protoGenerationTask)
+    dependsOn(jsonModelGenerationTask)
+
+    val outputDir = File("${project.buildDir}/rpm")
+
+    inputs.dir("cmd/cb-event-forwarder")
+    inputs.dir("cmd/go-serviced")
+    inputs.dir("pkg")
+    inputs.dir("scripts/")
+    inputs.files("cb-event-forwarder.rpm.spec", "MANIFEST*", "Makefile")
+    outputs.dir(outputDir)
+
+    doFirst {
+        project.delete(outputDir)
+    }
+
+    environment("RPM_OUTPUT_DIR", outputDir)
+    environment("GOPATH", goPath)
+    environment("RABBITMQ_SALT", rabbitMQSalt)
+    commandLine = listOf("make", "rpm")
+}
+
+val build = tasks.named("build").configure {
+    dependsOn(buildEventForwarderTask)
+}
+
 val unitTestTask = tasks.register<Exec>("runUnitTests") {
     dependsOn(protoGenerationTask)
     dependsOn(depTask)
+    dependsOn(buildEventForwarderTask)
 
     val unitTestResultsFile = File("$buildDir/unittest.out")
 
@@ -111,35 +140,6 @@ val criticTask = tasks.register<Exec>("criticizeCode") {
     args("critic")
 }
 
-val buildEventForwarderTask = tasks.register<Exec>("buildEventForwarder") {
-    dependsOn(depTask)
-    dependsOn(protoGenerationTask)
-    dependsOn(jsonModelGenerationTask)
-    dependsOn(unitTestTask)
-
-    val outputDir = File("${project.buildDir}/rpm")
-
-    inputs.dir("cmd/cb-event-forwarder")
-    inputs.dir("cmd/go-serviced")
-    inputs.dir("pkg")
-    inputs.dir("scripts/")
-    inputs.files("cb-event-forwarder.rpm.spec", "MANIFEST*", "Makefile")
-    outputs.dir(outputDir)
-
-    doFirst {
-        project.delete(outputDir)
-    }
-
-    environment("RPM_OUTPUT_DIR", outputDir)
-    environment("GOPATH", goPath)
-    environment("RABBITMQ_SALT", rabbitMQSalt)
-    commandLine = listOf("make", "rpm")
-}
-
-val build = tasks.named("build").configure {
-    dependsOn(buildEventForwarderTask)
-}
-
 val buildEventForwarderDockerImageTask = tasks.register<Exec>("buildEventForwarderDockerImage") {
     dependsOn(buildEventForwarderTask)
     executable("docker")

diff --git a/cb-event-forwarder.rpm.spec b/cb-event-forwarder.rpm.spec
@@ -3,7 +3,7 @@
 %global debug_package %{nil}
 %global __os_install_post /usr/lib/rpm/brp-compress %{nil}
 
-%define bare_version 3.8.3
+%define bare_version 3.8.4
 
 %define release 1
 

diff --git a/cmd/cb-event-forwarder/main.go b/cmd/cb-event-forwarder/main.go
@@ -29,7 +29,7 @@ var (
 	debug              = flag.Bool("debug", false, "Enable debugging mode")
 )
 
-var version = "3.8.3"
+var version = "3.8.4"
 var rabbitMQSalt = ""
 
 var signals = make(chan os.Signal, 2)

diff --git a/docker/compose.yml b/docker/compose.yml
@@ -4,9 +4,9 @@ services:
     image: projects.registry.vmware.com/carbonblack/event-forwarder:latest
     volumes:
       - ./data/config/cb.conf:/etc/cb/cb.conf:ro
-      - ./data/integrations:/etc/cb/integrations
+      - ./data/integrations:/etc/cb/integrations:z
       - ./data/config/integrations/event-forwarder:/root/event-forwarder-edr
-      - ./data/logs/event-forwarder:/var/log/cb/integrations/cb-event-forwarder
+      - ./data/logs/event-forwarder:/var/log/cb/integrations/cb-event-forwarder:z
     networks:
       - carbonblack
 networks:

diff --git a/gradle.properties b/gradle.properties
@@ -1,5 +1,5 @@
 # The event-forwarder version
-currentVersion=3.8.3-1
+currentVersion=3.8.4-1
 
 # Built in Gradle properties
 org.gradle.parallel=true
diff --git a/pkg/forwarder/input_worker.go b/pkg/forwarder/input_worker.go
@@ -20,7 +20,7 @@ import (
 )
 
 func (inputWorker InputWorker) processZipPB(body []byte, routingKey, contentType string, headers amqp.Table, exchangeName string) {
-	msgs, err := inputWorker.ProcessProtobufBundle(routingKey, body, headers)
+	msgs, err := inputWorker.ProcessRawZipBundle(routingKey, body, headers)
 	if err != nil {
 		inputWorker.reportBundleDetails(routingKey, body, headers)
 		inputWorker.reportError(routingKey, "Could not process raw zip bundle", err)

diff --git a/pkg/protobufmessageprocessor/new_protobuf_message_processor.go b/pkg/protobufmessageprocessor/new_protobuf_message_processor.go
@@ -861,6 +861,8 @@ func (pbm ProtobufMessageProcessor) ProcessProtobufBundle(routingKey string, bod
 		bytesRead += 4
 
 		if messageLength+bytesRead > totalLength {
+			log.Debugf("error in ProcessProtobufBundle for event index %d: Length %d exceeds %d; giving up - host: %s",
+				i, messageLength, totalLength, headers["sensorHostName"])
 			err = fmt.Errorf("error in ProcessProtobufBundle for event index %d: Length %d exceeds %d; giving up",
 				i, messageLength, totalLength)
 			break

diff --git a/test/raw_data/zip/0.txt b/test/raw_data/zip/0.txt
diff --git a/test/raw_data/zip/1.txt b/test/raw_data/zip/1.txt
diff --git a/test/raw_data/zip/10.txt b/test/raw_data/zip/10.txt
diff --git a/test/raw_data/zip/11.txt b/test/raw_data/zip/11.txt
diff --git a/test/raw_data/zip/12.txt b/test/raw_data/zip/12.txt
diff --git a/test/raw_data/zip/13.txt b/test/raw_data/zip/13.txt
diff --git a/test/raw_data/zip/14.txt b/test/raw_data/zip/14.txt
diff --git a/test/raw_data/zip/2.txt b/test/raw_data/zip/2.txt
diff --git a/test/raw_data/zip/3.txt b/test/raw_data/zip/3.txt
diff --git a/test/raw_data/zip/4.txt b/test/raw_data/zip/4.txt
diff --git a/test/raw_data/zip/5.txt b/test/raw_data/zip/5.txt
diff --git a/test/raw_data/zip/6.txt b/test/raw_data/zip/6.txt
diff --git a/test/raw_data/zip/7.txt b/test/raw_data/zip/7.txt
diff --git a/test/raw_data/zip/8.txt b/test/raw_data/zip/8.txt
diff --git a/test/raw_data/zip/9.txt b/test/raw_data/zip/9.txt