From 72bfb24b0d553e7e40da5ee735df0ad8472f2906 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Mon, 28 Mar 2022 17:30:40 -0600 Subject: [PATCH] feat: added --dryRun for validation --- internal/utility/json.go | 34 ++++++++++++++++++++++------------ main.go | 8 ++++++++ strategies/csv/csv.go | 10 +++++++--- strategies/legacy/context.go | 11 +++++++++++ strategies/legacy/legacy.go | 16 ++++++++-------- 5 files changed, 56 insertions(+), 23 deletions(-) diff --git a/internal/utility/json.go b/internal/utility/json.go index 7861fb5..a9fc587 100644 --- a/internal/utility/json.go +++ b/internal/utility/json.go @@ -9,26 +9,38 @@ import ( "github.com/pkg/errors" ) -func NewJSONWriter(fileName string) (*JSONWriter, error) { - f, err := os.Create(fileName) - if err != nil { - return nil, errors.Wrap(err, "could not create file for writing") +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { return nil } + +func NewJSONWriter(dryRun bool, fileName string) (*JSONWriter, error) { + var dest io.WriteCloser + if dryRun { + dest = nopCloser{io.Discard} + } else { + var err error + dest, err = os.Create(fileName) + if err != nil { + return nil, errors.Wrap(err, "could not create file for writing") + } } - w := bufio.NewWriter(f) + w := bufio.NewWriter(dest) return &JSONWriter{ - f: f, + f: dest, w: w, filename: fileName, }, nil } type JSONWriter struct { - f *os.File - w *bufio.Writer - documents uint64 - filename string + f io.WriteCloser + w *bufio.Writer + + filename string } func (c *JSONWriter) Write(doc easyjson.Marshaler) error { @@ -40,8 +52,6 @@ func (c *JSONWriter) Write(doc easyjson.Marshaler) error { return errors.Wrap(err, "could not write newline") } - c.documents++ - return nil } diff --git a/main.go b/main.go index 196fc3f..f9bf8d8 100644 --- a/main.go +++ b/main.go @@ -125,6 +125,10 @@ func main() { Usage: "folder where the outputted mongo files should be placed", Required: true, }, + cli.BoolFlag{ + Name: "dryRun", + Usage: "processes data to validate inputs without actually writing files", + }, }, }, { @@ -193,6 +197,10 @@ func main() { Usage: "folder where the outputted mongo files should be placed", Required: true, }, + cli.BoolFlag{ + Name: "dryRun", + Usage: "processes data to validate inputs without actually writing files", + }, }, }, } diff --git a/strategies/csv/csv.go b/strategies/csv/csv.go index b83f32c..305fdfb 100644 --- a/strategies/csv/csv.go +++ b/strategies/csv/csv.go @@ -54,6 +54,10 @@ func Import(c strategies.Context) error { // from the MongoDB export. input := c.String("input") + // dryRun indicates that the strategy should not write files and is used for + // validation. + dryRun := c.Bool("dryRun") + // auth is the identifier for the type of authentication profiles to be // created for the users. auth := c.String("auth") @@ -189,7 +193,7 @@ func Import(c strategies.Context) error { startedCommentsAt := time.Now() logrus.Debug("processing comments") - commentsWriter, err := utility.NewJSONWriter(commentsOutputFileName) + commentsWriter, err := utility.NewJSONWriter(dryRun, commentsOutputFileName) if err != nil { return errors.Wrap(err, "could not create comment writer") } @@ -296,7 +300,7 @@ func Import(c strategies.Context) error { startedUsersAt := time.Now() logrus.Debug("processing users") - usersWriter, err := utility.NewJSONWriter(usersOutputFileName) + usersWriter, err := utility.NewJSONWriter(dryRun, usersOutputFileName) if err != nil { return errors.Wrap(err, "could not create users writer") } @@ -414,7 +418,7 @@ func Import(c strategies.Context) error { startedStoriesAt := time.Now() logrus.Debug("processing stories") - storiesWriter, err := utility.NewJSONWriter(storiesOutputFileName) + storiesWriter, err := utility.NewJSONWriter(dryRun, storiesOutputFileName) if err != nil { return errors.Wrap(err, "could not create story writer") } diff --git a/strategies/legacy/context.go b/strategies/legacy/context.go index 908d638..f166084 100644 --- a/strategies/legacy/context.go +++ b/strategies/legacy/context.go @@ -6,6 +6,7 @@ import ( "github.com/coralproject/coral-importer/common" "github.com/coralproject/coral-importer/common/coral" "github.com/coralproject/coral-importer/strategies" + "github.com/sirupsen/logrus" ) type CommentRef struct { @@ -41,7 +42,16 @@ func NewContext(c strategies.Context) *Context { // from the MongoDB export. input := c.String("input") + // dryRun indicates that the strategy should not write files and is used for + // validation. + dryRun := c.Bool("dryRun") + + if dryRun { + logrus.Warn("dry run is enabled, files will not be written") + } + return &Context{ + DryRun: dryRun, TenantID: tenantID, SiteID: siteID, Filenames: Filenames{ @@ -85,6 +95,7 @@ type Filenames struct { } type Context struct { + DryRun bool TenantID string SiteID string Filenames Filenames diff --git a/strategies/legacy/legacy.go b/strategies/legacy/legacy.go index 3d83d28..401f980 100644 --- a/strategies/legacy/legacy.go +++ b/strategies/legacy/legacy.go @@ -133,13 +133,13 @@ func SeedCommentsReferences(ctx *Context) error { } func WriteCommentActions(ctx *Context) error { - commentActionsWriter, err := utility.NewJSONWriter(ctx.Filenames.Output.CommentActions) + commentActionsWriter, err := utility.NewJSONWriter(ctx.DryRun, ctx.Filenames.Output.CommentActions) if err != nil { return errors.Wrap(err, "could not create commentActionsWriter") } defer commentActionsWriter.Close() - bar, err := utility.NewLineCounter("Writing Comment Actions", ctx.Filenames.Input.Actions) + bar, err := utility.NewLineCounter("Comment Actions", ctx.Filenames.Input.Actions) if err != nil { return errors.Wrap(err, "could not count actions file") } @@ -199,13 +199,13 @@ func WriteCommentActions(ctx *Context) error { } func WriteComments(ctx *Context) error { - commentsWriter, err := utility.NewJSONWriter(ctx.Filenames.Output.Comments) + commentsWriter, err := utility.NewJSONWriter(ctx.DryRun, ctx.Filenames.Output.Comments) if err != nil { return errors.Wrap(err, "could not create comments writer") } defer commentsWriter.Close() - bar, err := utility.NewLineCounter("Writing Comments", ctx.Filenames.Input.Comments) + bar, err := utility.NewLineCounter("Comments", ctx.Filenames.Input.Comments) if err != nil { return errors.Wrap(err, "could not count comments file") } @@ -266,13 +266,13 @@ func WriteComments(ctx *Context) error { } func WriteStories(ctx *Context) error { - storiesWriter, err := utility.NewJSONWriter(ctx.Filenames.Output.Stories) + storiesWriter, err := utility.NewJSONWriter(ctx.DryRun, ctx.Filenames.Output.Stories) if err != nil { return errors.Wrap(err, "could not create stories writer") } defer storiesWriter.Close() - bar, err := utility.NewLineCounter("Writing Stories", ctx.Filenames.Input.Assets) + bar, err := utility.NewLineCounter("Stories", ctx.Filenames.Input.Assets) if err != nil { return errors.Wrap(err, "could not count assets file") } @@ -322,13 +322,13 @@ func WriteStories(ctx *Context) error { } func WriteUsers(ctx *Context) error { - usersWriter, err := utility.NewJSONWriter(ctx.Filenames.Output.Users) + usersWriter, err := utility.NewJSONWriter(ctx.DryRun, ctx.Filenames.Output.Users) if err != nil { return errors.Wrap(err, "could not create users writer") } defer usersWriter.Close() - bar, err := utility.NewLineCounter("Writing Users", ctx.Filenames.Input.Users) + bar, err := utility.NewLineCounter("Users", ctx.Filenames.Input.Users) if err != nil { return errors.Wrap(err, "could not count users file") }