[7.0.x] Include gravity resources in debug report

lib/ops/opsservice/report.go

@@ -159,6 +159,9 @@ func (s *site) getReport(ctx context.Context, runner remoteRunner, servers []rem
 		if err := s.collectStatusTimeline(reportWriter, serverRunner); err != nil {
 			logger.WithError(err).Error("Failed to collect status timeline.")
 		}
+		if err := s.collectResources(reportWriter, serverRunner); err != nil {
+			logger.WithError(err).Error("Failed to collect gravity resources.")
+		}
 	}
 
 	// use a pipe to avoid allocating a buffer
@@ -288,6 +291,21 @@ func (s *site) collectStatusTimeline(reportWriter report.FileWriter, runner *ser
 	return nil
 }
 
+func (s *site) collectResources(reportWriter report.FileWriter, runner *serverRunner) error {
+	w, err := reportWriter.NewWriter("resources.tar.gz")
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	defer w.Close()
+	var stderr bytes.Buffer
+	err = runner.RunStream(w, &stderr, s.gravityCommand("system", "report",
+		fmt.Sprintf("--filter=%v", report.FilterResources), "--compressed")...)
+	if err != nil {
+		return trace.Wrap(err, "failed to collect gravity resources: %s", stderr.String())
+	}
+	return nil
+}
+
 func runCollectors(ctx context.Context, cluster site, dir string) error {
 	storageSite, err := cluster.service.cfg.Backend.GetSite(cluster.domainName)
 	if err != nil {

lib/report/report.go

@@ -52,6 +52,8 @@ func Collect(ctx context.Context, config Config, w io.Writer) error {
 			collectors = append(collectors, etcdMetrics()...)
 		case FilterTimeline:
 			collectors = append(collectors, NewTimelineCollector())
+		case FilterResources:
+			collectors = append(collectors, ResourceCollectors()...)
 		}
 	}
 
@@ -124,7 +126,10 @@ const (
 
 	// FilterTimeline defines a report collection filter to fetch the status timeline
 	FilterTimeline = "timeline"
+
+	// FilterResources defines a report collection filter to fetch gravity resources
+	FilterResources = "resources"
 )
 
 // AllFilters lists all available collector filters
-var AllFilters = []string{FilterSystem, FilterKubernetes, FilterEtcd, FilterTimeline}
+var AllFilters = []string{FilterSystem, FilterKubernetes, FilterEtcd, FilterTimeline, FilterResources}

lib/report/resources.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2020 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package report
+
+import (
+	"fmt"
+
+	"github.com/gravitational/gravity/lib/storage"
+	"github.com/gravitational/gravity/lib/utils"
+)
+
+// ResourceCollectors returns gravity resource collectors.
+func ResourceCollectors() Collectors {
+	// Collect select gravity resources. More information on supported gravity
+	// resources found at https://gravitational.com/gravity/docs/config/
+	resources := []string{
+		storage.KindClusterConfiguration,
+		storage.KindRuntimeEnvironment,
+		storage.KindAuthGateway,
+		storage.KindOperation,
+		storage.KindSMTPConfig,
+		storage.KindAlertTarget,
+		storage.KindAlert,
+		storage.KindLogForwarder,
+	}
+
+	collectors := make(Collectors, len(resources))
+	for i, resource := range resources {
+		collectors[i] = Cmd(fmt.Sprintf("%s.yaml", resource), gravityResourceYAML(resource)...)
+	}
+	return collectors
+}
+
+// gravityResourceYAML returns the gravity command to output the specified
+// resource in YAML format.
+func gravityResourceYAML(resource string) []string {
+	return utils.Self("resource", "get", "--format", "yaml", resource)
+}

lib/report/system.go

@@ -40,9 +40,9 @@ func NewSystemCollector(since time.Duration) Collectors {
 
 	add(basicSystemInfo()...)
 	add(systemStatus()...)
+	add(syslogExportLogs(since)...)
 	add(systemFileLogs()...)
 	add(planetLogs(since)...)
-	add(syslogExportLogs(since))
 	add(auditLog())
 	add(gravityCLILog(since))
 
@@ -124,15 +124,21 @@ func systemStatus() Collectors {
 
 // syslogExportLogs fetches logs for gravity binary invocations
 // (including installation logs)
-func syslogExportLogs(since time.Duration) Collector {
+func syslogExportLogs(since time.Duration) Collectors {
 	var script = `
 #!/bin/bash
-/bin/journalctl --no-pager --output=export`
+/bin/journalctl --no-pager `
 	if since != 0 {
 		script = fmt.Sprintf(`%s --since="%s" `, script, time.Now().Add(-since).Format(JournalDateFormat))
 	}
-	script = fmt.Sprintf("%s | /bin/gzip -f", script)
-	return Script("gravity-journal.log.gz", script)
+
+	plain := fmt.Sprintf("%s | /bin/gzip -f", script)
+	export := fmt.Sprintf("%s --output=export | /bin/gzip -f", script)
+
+	return Collectors{
+		Script("gravity-journal.log.gz", plain),
+		Script("gravity-journal-export.log.gz", export),
+	}
 }
 
 // systemFileLogs fetches gravity platform-related logs
@@ -152,12 +158,17 @@ cat %v 2> /dev/null || true`
 // planetLogs fetches planet syslog messages as well as the fresh journal entries
 func planetLogs(since time.Duration) Collectors {
 	return Collectors{
+		Self("planet-journal.log.gz",
+			"system", "export-runtime-journal",
+			"--since", since.String()),
 		// Fetch planet journal entries for the last two days
 		// The log can be imported as a journal with systemd-journal-remote:
 		//
 		// $ cat ./node-1-planet-journal-export.log | /lib/systemd/systemd-journal-remote -o ./journal/system.journal -
 		Self("planet-journal-export.log.gz",
-			"system", "export-runtime-journal", "--since", since.String()),
+			"system", "export-runtime-journal",
+			"--since", since.String(),
+			"--export"),
 	}
 }
 

tool/gravity/cli/commands.go

@@ -1714,6 +1714,8 @@ type SystemExportRuntimeJournalCmd struct {
 	// filter. Only log entries from the start of the time filter until now will
 	// be included in the report.
 	Since *time.Duration
+	// Export serializes the journal into a binary stream.
+	Export *bool
 }
 
 // SystemStreamRuntimeJournalCmd streams contents of the runtime journal
@@ -1723,6 +1725,8 @@ type SystemStreamRuntimeJournalCmd struct {
 	// filter. Only log entries from the start of the time filter until now will
 	// be included in the report.
 	Since *time.Duration
+	// Export serializes the journal into a binary stream.
+	Export *bool
 }
 
 // SystemSelinuxBootstrapCmd configures SELinux file contexts and ports on the node

tool/gravity/cli/journal.go

@@ -40,7 +40,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func exportRuntimeJournal(env *localenv.LocalEnvironment, outputFile string, since time.Duration) error {
+func exportRuntimeJournal(env *localenv.LocalEnvironment, outputFile string, since time.Duration, export bool) error {
 	stateDir, err := state.GetStateDir()
 	if err != nil {
 		return trace.Wrap(err)
@@ -92,15 +92,21 @@ func exportRuntimeJournal(env *localenv.LocalEnvironment, outputFile string, sin
 
 	zip := gzip.NewWriter(w)
 	defer zip.Close()
-	cmd := exec.CommandContext(ctx, utils.Exe.Path,
+
+	args := []string{
 		"system", "stream-runtime-journal",
-		"--since", since.String())
+		"--since", since.String(),
+	}
+	if export {
+		args = append(args, "--export")
+	}
+	cmd := exec.CommandContext(ctx, utils.Exe.Path, args...)
 	cmd.Stdout = zip
 	cmd.Stderr = zip
 	return trace.Wrap(cmd.Run())
 }
 
-func streamRuntimeJournal(env *localenv.LocalEnvironment, since time.Duration) error {
+func streamRuntimeJournal(env *localenv.LocalEnvironment, since time.Duration, export bool) error {
 	runtimePackage, err := pack.FindRuntimePackage(env.Packages)
 	if err != nil {
 		return trace.Wrap(err)
@@ -124,9 +130,11 @@ func streamRuntimeJournal(env *localenv.LocalEnvironment, since time.Duration) e
 	const cmd = defaults.JournalctlBinHost
 	args := []string{
 		cmd,
-		"--output", "export",
 		"-D", journalDir,
 	}
+	if export {
+		args = append(args, "--output", "export")
+	}
 	if since != 0 {
 		args = append(args, "--since", time.Now().Add(-since).Format(report.JournalDateFormat))
 	}

tool/gravity/cli/register.go

@@ -734,9 +734,11 @@ func RegisterCommands(app *kingpin.Application) *Application {
 	g.SystemExportRuntimeJournalCmd.CmdClause = g.SystemCmd.Command("export-runtime-journal", "Export runtime journal logs to a file").Hidden()
 	g.SystemExportRuntimeJournalCmd.OutputFile = g.SystemExportRuntimeJournalCmd.Flag("output", "Name of resulting tarball. Output to stdout if unspecified").String()
 	g.SystemExportRuntimeJournalCmd.Since = g.SystemExportRuntimeJournalCmd.Flag("since", "Only return logs newer than a relative duration like 5s, 2m, or 3h. Default is 336h (14 days). Specify 0s to collect all logs.").Default("336h").Duration()
+	g.SystemExportRuntimeJournalCmd.Export = g.SystemExportRuntimeJournalCmd.Flag("export", "Serializes the journal into a binary stream").Bool()
 
 	g.SystemStreamRuntimeJournalCmd.CmdClause = g.SystemCmd.Command("stream-runtime-journal", "Stream runtime journal to stdout").Hidden()
 	g.SystemStreamRuntimeJournalCmd.Since = g.SystemStreamRuntimeJournalCmd.Flag("since", "Only return logs newer than a relative duration like 5s, 2m, or 3h. Default is 336h (14 days). Specify 0s to collect all logs.").Default("336h").Duration()
+	g.SystemStreamRuntimeJournalCmd.Export = g.SystemStreamRuntimeJournalCmd.Flag("export", "Serializes the journal into a binary stream").Bool()
 
 	g.SystemSelinuxBootstrapCmd.CmdClause = g.SystemCmd.Command("selinux-bootstrap", "Configure SELinux file contexts and ports on the node")
 	g.SystemSelinuxBootstrapCmd.Path = g.SystemSelinuxBootstrapCmd.Flag("output", "Path to output file for bootstrap script").String()

tool/gravity/cli/run.go

@@ -942,10 +942,12 @@ func Execute(g *Application, cmd string, extraArgs []string) (err error) {
 	case g.SystemExportRuntimeJournalCmd.FullCommand():
 		return exportRuntimeJournal(localEnv,
 			*g.SystemExportRuntimeJournalCmd.OutputFile,
-			*g.SystemExportRuntimeJournalCmd.Since)
+			*g.SystemExportRuntimeJournalCmd.Since,
+			*g.SystemExportRuntimeJournalCmd.Export)
 	case g.SystemStreamRuntimeJournalCmd.FullCommand():
 		return streamRuntimeJournal(localEnv,
-			*g.SystemStreamRuntimeJournalCmd.Since)
+			*g.SystemStreamRuntimeJournalCmd.Since,
+			*g.SystemStreamRuntimeJournalCmd.Export)
 	case g.SystemSelinuxBootstrapCmd.FullCommand():
 		return bootstrapSELinux(localEnv,
 			*g.SystemSelinuxBootstrapCmd.Path,