From 315b594bae52f2f710f7b8487558651f39e5e71c Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Wed, 25 Oct 2023 02:01:58 +0000 Subject: [PATCH 1/2] Remove references to removed bosh-lite scripts These scripts were removed in c96846d7. --- README.md | 10 ++-------- tests/README.md | 18 +++--------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index e2563dbf1..af624e0f1 100644 --- a/README.md +++ b/README.md @@ -224,14 +224,8 @@ will need to [install Go][go-installation]. Note that the `go.mod` and `go.sum` files in this repo are only for the test code. -Before running tests, you'll need to create a bosh director. First, ensure the -bosh cli is installed and the bosh-deployment repository is downloaded and -located at `~/workspace/bosh-deployment`. You can then run -`./scripts/setup-bosh-lite-for-tests.sh` to create the director. Afterwards -execute `source export-bosh-lite-creds.sh` to target the bosh director. - -(Alternatively, if you wish to run the specs against the CI env, you can -`source` the `.envrc` in your env-repo.) +The tests expect that your local environment has the BOSH CLI installed and +configured correctly to point to a BOSH Director. The tests can then be run from the top of the repo with `./scripts/test`. diff --git a/tests/README.md b/tests/README.md index 37e2f3067..7dda48853 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,24 +3,12 @@ ## Development The following commands assume you are operating -from the top level of the repo, have go inistalled, +from the top level of the repo, have go installed, and have initialized and updated the blackbox submodule as described in the main [README](../README.md). -First, you'll need to setup a bosh-lite and login to it. -If you don't have a bosh-lite running -and aliased as `vbox` already: - -```sh -scripts/setup-bosh-lite-for-tests.sh -``` - -If you don't already have BOSH credential -environment variables in your session: - -```sh -source scripts/export-bosh-lite-creds.sh -``` +Ensure that you have a BOSH Director deployed and your local environment is +configured to point to the BOSH Director. To then run the tests locally: From a70f103294bdc6dc922d90ee79543d873dca51ce Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Wed, 25 Oct 2023 01:04:32 +0000 Subject: [PATCH 2/2] Include optional environment identifier - Operators are already able to tag emitted log lines with the BOSH Director name with the `director` structured data parameter. - Add a new `environment` structured data parameter for cases where log lines should be tagged with an operator provided string but need to vary from the director name. - Don't add the environment SD-PARAM when the BOSH property is not provided by the operator. This prevents any log parsing breakage that might occur if log processors were to be string matching rather than parsing syslog messages correctly. --- README.md | 3 ++ jobs/syslog_forwarder/spec | 4 +- .../syslog-release-forwarding-setup.conf.erb | 7 ++++ tests/acceptance_test.go | 40 +++++++++++++++++++ tests/manifests/environment-identifier.yml | 40 +++++++++++++++++++ 5 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 tests/manifests/environment-identifier.yml diff --git a/README.md b/README.md index af624e0f1..2cf5a6d90 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,9 @@ contains the following fields: * instance group * instance ID +The structured data may also contain an optional `environment` field if +provided by the operator. + The whole thing looks something like this: ``` diff --git a/jobs/syslog_forwarder/spec b/jobs/syslog_forwarder/spec index ee9e9d1ea..2fab16baf 100644 --- a/jobs/syslog_forwarder/spec +++ b/jobs/syslog_forwarder/spec @@ -29,7 +29,9 @@ properties: syslog.director: description: "BOSH Director name" default: "" - + syslog.environment: + description: "Optional environment identifier" + default: "" syslog.forward_files: description: If enabled, use BlackBox to forward logs. default: true diff --git a/jobs/syslog_forwarder/templates/syslog-release-forwarding-setup.conf.erb b/jobs/syslog_forwarder/templates/syslog-release-forwarding-setup.conf.erb index df28db12e..e02c0c989 100644 --- a/jobs/syslog_forwarder/templates/syslog-release-forwarding-setup.conf.erb +++ b/jobs/syslog_forwarder/templates/syslog-release-forwarding-setup.conf.erb @@ -18,6 +18,9 @@ end %> set $.director = "<%= p('syslog.director') %>"; +<% if p('syslog.environment') != '' -%> +set $.environment = "<%= p('syslog.environment') %>"; +<% end -%> set $.deployment = "<%= spec.deployment %>"; set $.instance = "<%= spec.name %>"; set $.az = "<%= spec.az %>"; @@ -37,6 +40,10 @@ template(name="SyslogForwarderTemplate" type="list") { # 47450 is CFF in https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers constant(value=" [instance@47450 director=\"") property(name="$.director") +<% if p('syslog.environment') != '' -%> + constant(value="\" environment=\"") + property(name="$.environment") +<% end -%> constant(value="\" deployment=\"") property(name="$.deployment") constant(value="\" group=\"") diff --git a/tests/acceptance_test.go b/tests/acceptance_test.go index 6c9982fb4..945777323 100644 --- a/tests/acceptance_test.go +++ b/tests/acceptance_test.go @@ -99,6 +99,7 @@ var _ = Describe("Forwarding loglines to a TCP syslog drain", func() { Expect(string(sdata.ID())).To(Equal("instance@47450")) properties := sdata.Properties() Expect(properties).To(ContainElement(logrfc.Property{Key: ("director"), Value: ("")})) + Expect(properties).ToNot(ContainElement(logrfc.Property{Key: ("environment"), Value: ("")})) Expect(properties).To(ContainElement(logrfc.Property{Key: ("deployment"), Value: (DeploymentName())})) Expect(properties).To(ContainElement(logrfc.Property{Key: ("group"), Value: ("forwarder")})) break @@ -246,6 +247,45 @@ var _ = Describe("Forwarding loglines to a TCP syslog drain", func() { }) }) +var _ = Describe("Tagging logs with environment identifiers", func() { + AfterEach(func() { + Cleanup() + }) + + Context("when an environment identifier is specified", func() { + BeforeEach(func() { + Cleanup() + Deploy("manifests/environment-identifier.yml") + }) + It("includes the environment identifier in the structured data", func() { + SendLogMessage("test-environment-identifier", DEFAULT_LOGGER_SIZE) + Eventually(func() string { + return ForwardedLogs() + }).Should(ContainSubstring("test-environment-identifier")) + + logs := bytes.NewBufferString(ForwardedLogs()) + reader := bufio.NewReader(logs) + + for { + line, _, err := reader.ReadLine() + Expect(err).ToNot(HaveOccurred()) + if len(line) == 0 { + break + } + logLine, _, err := logrfc.Parse(line) + Expect(err).ToNot(HaveOccurred()) + if string(logLine.Message()) == "test-environment-identifier" { + sdata := logLine.StructureData()[0] + Expect(string(sdata.ID())).To(Equal("instance@47450")) + properties := sdata.Properties() + Expect(properties).To(ContainElement(logrfc.Property{Key: ("environment"), Value: ("some-environment-identifier")})) + break + } + } + }) + }) +}) + var _ = Describe("Optional features to reduce CF log volume", func() { AfterEach(func() { Cleanup() diff --git a/tests/manifests/environment-identifier.yml b/tests/manifests/environment-identifier.yml new file mode 100644 index 000000000..01d4ab910 --- /dev/null +++ b/tests/manifests/environment-identifier.yml @@ -0,0 +1,40 @@ +--- +name: ((deployment)) +releases: + - name: syslog + version: latest +stemcells: + - alias: default + os: ((stemcell-os)) + version: latest +instance_groups: + - name: forwarder + instances: 1 + vm_type: default + stemcell: default + networks: + - name: default + azs: + - z1 + jobs: + - name: syslog_forwarder + release: syslog + properties: + syslog: + environment: "some-environment-identifier" + - name: storer + instances: 1 + vm_type: default + stemcell: default + networks: + - name: default + azs: + - z1 + jobs: + - name: syslog_storer + release: syslog +update: + canaries: 1 + max_in_flight: 1 + canary_watch_time: 1000-60000 + update_watch_time: 1000-60000