CLOUDP-73987: [mongocli] Improve log downloads (#468)

examples/atlas/log-download.md

@@ -0,0 +1,9 @@
+# Download and Unzip Logs
+
+You can download and unzip logs in one go like
+
+```bash
+mongocli atlas logs download <processName> mongodb.gz \
+    --out /dev/stdout \
+    --force | gunzip
+```

internal/cli/atlas/logs/download.go

@@ -16,8 +16,6 @@ package logs
 
 import (
 	"fmt"
-	"io"
-	"os"
 	"strings"
 
 	"github.com/mongodb/mongocli/internal/cli"
@@ -33,12 +31,11 @@ import (
 
 type DownloadOpts struct {
 	cli.GlobalOpts
+	cli.DownloaderOpts
 	host  string
 	name  string
-	out   string
 	start string
 	end   string
-	fs    afero.Fs
 	store store.LogsDownloader
 }
 
@@ -51,38 +48,26 @@ func (opts *DownloadOpts) initStore() error {
 }
 
 func (opts *DownloadOpts) Run() error {
-	f, err := opts.newWriteCloser()
+	f, err := opts.NewWriteCloser()
 	if err != nil {
 		return err
 	}
 
 	r := opts.newDateRangeOpts()
 	if err := opts.store.DownloadLog(opts.ConfigProjectID(), opts.host, opts.name, f, r); err != nil {
-		_ = opts.handleError(f)
+		_ = opts.OnError(f)
 		return err
 	}
-
-	fmt.Printf(downloadMessage, opts.out)
+	if opts.Out != "/dev/stdout" {
+		fmt.Printf(downloadMessage, opts.Out)
+	}
 	return f.Close()
 }
 
-func (opts *DownloadOpts) output() string {
-	if opts.out == "" {
-		opts.out = strings.ReplaceAll(opts.name, ".gz", ".log.gz")
+func (opts *DownloadOpts) initDefaultOut() {
+	if opts.Out == "" {
+		opts.Out = strings.ReplaceAll(opts.name, ".gz", ".log.gz")
 	}
-	return opts.out
-}
-
-func (opts *DownloadOpts) handleError(f io.Closer) error {
-	_ = f.Close()
-	return opts.fs.Remove(opts.output())
-}
-
-func (opts *DownloadOpts) newWriteCloser() (io.WriteCloser, error) {
-	// Create file only if is not there already (don't overwrite)
-	ff := os.O_CREATE | os.O_TRUNC | os.O_WRONLY | os.O_EXCL
-	f, err := opts.fs.OpenFile(opts.output(), ff, 0777)
-	return f, err
 }
 
 func (opts *DownloadOpts) newDateRangeOpts() *atlas.DateRangetOptions {
@@ -92,36 +77,36 @@ func (opts *DownloadOpts) newDateRangeOpts() *atlas.DateRangetOptions {
 	}
 }
 
-var logNames = []string{"mongodb.gz", "mongos.gz", "mongodb-audit-log.gz", "mongos-audit-log.gz"}
-
 // mongocli atlas logs download <hostname> <logname> [--type type] [--output destination] [--projectId projectId]
 func DownloadBuilder() *cobra.Command {
 	const argsN = 2
-	opts := &DownloadOpts{
-		fs: afero.NewOsFs(),
-	}
+	opts := &DownloadOpts{}
+	opts.Fs = afero.NewOsFs()
 	cmd := &cobra.Command{
 		Use:   "download <hostname> <logname>",
 		Short: download,
 		Long:  downloadLong,
 		Args:  cobra.ExactArgs(argsN),
 		PreRunE: func(cmd *cobra.Command, args []string) error {
+			opts.initDefaultOut()
 			return opts.PreRunE(opts.initStore)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			opts.host = args[0]
 			opts.name = args[1]
-			if !search.StringInSlice(logNames, opts.name) {
-				return fmt.Errorf("<logname> must be one of %s", logNames)
+			if !search.StringInSlice(cmd.ValidArgs, opts.name) {
+				return fmt.Errorf("<logname> must be one of %s", cmd.ValidArgs)
 			}
 			return opts.Run()
 		},
+		ValidArgs: []string{"mongodb.gz", "mongos.gz", "mongodb-audit-log.gz", "mongos-audit-log.gz"},
 	}
 
-	cmd.Flags().StringVar(&opts.out, flag.Out, "", usage.LogOut)
-
+	cmd.Flags().StringVar(&opts.Out, flag.Out, "", usage.LogOut)
 	cmd.Flags().StringVar(&opts.start, flag.Start, "", usage.LogStart)
 	cmd.Flags().StringVar(&opts.end, flag.End, "", usage.LogEnd)
+	cmd.Flags().BoolVar(&opts.Force, flag.Force, false, usage.ForceFile)
+
 	cmd.Flags().StringVar(&opts.ProjectID, flag.ProjectID, "", usage.ProjectID)
 
 	_ = cmd.MarkFlagFilename(flag.Out)

internal/cli/atlas/logs/download_test.go

@@ -20,22 +20,24 @@ import (
 	"testing"
 
 	"github.com/golang/mock/gomock"
+	"github.com/mongodb/mongocli/internal/cli"
+	"github.com/mongodb/mongocli/internal/flag"
 	"github.com/mongodb/mongocli/internal/mocks"
 	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestLogsDownloadOpts_Run(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	mockStore := mocks.NewMockLogsDownloader(ctrl)
 	defer ctrl.Finish()
 
-	appFS := afero.NewMemMapFs()
-
 	opts := &DownloadOpts{
 		name:  "mongo.gz",
-		fs:    appFS,
 		store: mockStore,
 	}
+	opts.Out = opts.name
+	opts.Fs = afero.NewMemMapFs()
 
 	mockStore.
 		EXPECT().
@@ -47,3 +49,54 @@ func TestLogsDownloadOpts_Run(t *testing.T) {
 		t.Fatalf("Run() unexpected error: %v", err)
 	}
 }
+
+func TestDownloadBuilder(t *testing.T) {
+	cli.CmdValidator(
+		t,
+		DownloadBuilder(),
+		0,
+		[]string{flag.Out, flag.ProjectID, flag.Force, flag.Start, flag.End},
+	)
+}
+
+func TestDownloadOpts_initDefaultOut(t *testing.T) {
+	type fields struct {
+		logName string
+		out     string
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   string
+	}{
+		{
+			name: "empty out and add log",
+			fields: fields{
+				logName: "mongo.gz",
+				out:     "",
+			},
+			want: "mongo.log.gz",
+		},
+		{
+			name: "with out",
+			fields: fields{
+				logName: "mongo.gz",
+				out:     "myfile.gz",
+			},
+			want: "myfile.gz",
+		},
+	}
+	for _, tt := range tests {
+		logName := tt.fields.logName
+		out := tt.fields.out
+		want := tt.want
+		t.Run(tt.name, func(t *testing.T) {
+			opts := &DownloadOpts{
+				name: logName,
+			}
+			opts.Out = out
+			opts.initDefaultOut()
+			assert.Equal(t, opts.Out, want)
+		})
+	}
+}

internal/cli/cloudmanager/cloud_manager_test.go

@@ -1,3 +1,17 @@
+// Copyright 2020 MongoDB Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package cloudmanager
 
 import (

internal/cli/downloader_opts.go

@@ -25,13 +25,17 @@ import (
 // A command can compose this struct and then safely rely on the methods Prompt, or Delete
 // to manage the interactions with the user
 type DownloaderOpts struct {
-	Out string
-	Fs  afero.Fs
+	Out   string
+	Force bool
+	Fs    afero.Fs
 }
 
 func (opts *DownloaderOpts) NewWriteCloser() (io.WriteCloser, error) {
 	// Create file only if is not there already (don't overwrite)
-	ff := os.O_CREATE | os.O_TRUNC | os.O_WRONLY | os.O_EXCL
+	ff := os.O_CREATE | os.O_TRUNC | os.O_WRONLY
+	if !opts.Force {
+		ff |= os.O_EXCL
+	}
 	f, err := opts.Fs.OpenFile(opts.Out, ff, 0777)
 	return f, err
 }

internal/cli/opsmanager/backup/enable.go

@@ -1,3 +1,17 @@
+// Copyright 2020 MongoDB Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package backup
 
 import (

internal/cli/opsmanager/backup/enable_test.go

@@ -1,3 +1,17 @@
+// Copyright 2020 MongoDB Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package backup
 
 import (

internal/cli/opsmanager/logs/jobs.go

@@ -25,10 +25,12 @@ func JobsBuilder() *cobra.Command {
 		Short:   LogCollection,
 	}
 
-	cmd.AddCommand(JobsCollectOptsBuilder())
-	cmd.AddCommand(JobsListOptsBuilder())
-	cmd.AddCommand(JobsDownloadOptsBuilder())
-	cmd.AddCommand(JobsDeleteOptsBuilder())
+	cmd.AddCommand(
+		JobsCollectOptsBuilder(),
+		JobsListOptsBuilder(),
+		JobsDownloadOptsBuilder(),
+		JobsDeleteOptsBuilder(),
+	)
 
 	return cmd
 }

internal/cli/opsmanager/logs/jobs_download.go

@@ -71,6 +71,7 @@ func JobsDownloadOptsBuilder() *cobra.Command {
 	cmd.Flags().StringVar(&opts.Out, flag.Out, "", usage.LogOut)
 
 	cmd.Flags().StringVar(&opts.ProjectID, flag.ProjectID, "", usage.ProjectID)
+	cmd.Flags().BoolVar(&opts.Force, flag.Force, false, usage.ForceFile)
 
 	_ = cmd.MarkFlagRequired(flag.Out)
 	_ = cmd.MarkFlagFilename(flag.Out)

internal/cli/opsmanager/logs/jobs_download_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	"github.com/golang/mock/gomock"
+	"github.com/mongodb/mongocli/internal/cli"
+	"github.com/mongodb/mongocli/internal/flag"
 	"github.com/mongodb/mongocli/internal/mocks"
 	"github.com/spf13/afero"
 )
@@ -46,3 +48,12 @@ func TestDownloadOpts_Run(t *testing.T) {
 		t.Fatalf("Run() unexpected error: %v", err)
 	}
 }
+
+func TestJobsDownloadOptsBuilder(t *testing.T) {
+	cli.CmdValidator(
+		t,
+		JobsDownloadOptsBuilder(),
+		0,
+		[]string{flag.Out, flag.ProjectID, flag.Force},
+	)
+}

internal/cli/opsmanager/monitoring/enable.go

@@ -1,3 +1,17 @@
+// Copyright 2020 MongoDB Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitoring
 
 import (

internal/cli/opsmanager/monitoring/enable_test.go

@@ -1,3 +1,17 @@
+// Copyright 2020 MongoDB Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitoring
 
 import (

internal/usage/usage.go

@@ -77,11 +77,12 @@ const (
 	WhiteListsDeleteAfter           = "ISO-8601-formatted UTC date after which Atlas removes the entry from the whitelist."
 	BDUsersDeleteAfter              = "Timestamp in ISO 8601 date and time format in UTC after which Atlas deletes the user."
 	Force                           = "Don't ask for confirmation."
+	ForceFile                       = "Overwrite the destination file."
 	Email                           = "User’s email address."
 	LogOut                          = "Optional output filename, if none given will use the log name."
 	DiagnoseOut                     = "Optional output filename, if none given will use diagnose-archive.tar.gz."
-	LogStart                        = "Beginning of the period for which to retrieve logs."
-	LogEnd                          = "End of the period for which to retrieve logs."
+	LogStart                        = "Beginning of the period for which to retrieve logs in UNIX Epoch time."
+	LogEnd                          = "End of the period for which to retrieve logs in  UNIX Epoch time."
 	ArchiveLimit                    = "Max number of entries for the diagnose archive."
 	ArchiveMinutes                  = "Beginning of the period for which to retrieve diagnose archive. Ops Manager takes out minutes from the current time. "
 	MeasurementStart                = "Beginning of the period for which to retrieve measurements."