feat: added --dryRun for validation

internal/utility/json.go

@@ -9,26 +9,38 @@ import (
 	"github.com/pkg/errors"
 )
 
-func NewJSONWriter(fileName string) (*JSONWriter, error) {
-	f, err := os.Create(fileName)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create file for writing")
+type nopCloser struct {
+	io.Writer
+}
+
+func (nopCloser) Close() error { return nil }
+
+func NewJSONWriter(dryRun bool, fileName string) (*JSONWriter, error) {
+	var dest io.WriteCloser
+	if dryRun {
+		dest = nopCloser{io.Discard}
+	} else {
+		var err error
+		dest, err = os.Create(fileName)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create file for writing")
+		}
 	}
 
-	w := bufio.NewWriter(f)
+	w := bufio.NewWriter(dest)
 
 	return &JSONWriter{
-		f:        f,
+		f:        dest,
 		w:        w,
 		filename: fileName,
 	}, nil
 }
 
 type JSONWriter struct {
-	f         *os.File
-	w         *bufio.Writer
-	documents uint64
-	filename  string
+	f io.WriteCloser
+	w *bufio.Writer
+
+	filename string
 }
 
 func (c *JSONWriter) Write(doc easyjson.Marshaler) error {
@@ -40,8 +52,6 @@ func (c *JSONWriter) Write(doc easyjson.Marshaler) error {
 		return errors.Wrap(err, "could not write newline")
 	}
 
-	c.documents++
-
 	return nil
 }
 

main.go

@@ -125,6 +125,10 @@ func main() {
 					Usage:    "folder where the outputted mongo files should be placed",
 					Required: true,
 				},
+				cli.BoolFlag{
+					Name:  "dryRun",
+					Usage: "processes data to validate inputs without actually writing files",
+				},
 			},
 		},
 		{
@@ -193,6 +197,10 @@ func main() {
 					Usage:    "folder where the outputted mongo files should be placed",
 					Required: true,
 				},
+				cli.BoolFlag{
+					Name:  "dryRun",
+					Usage: "processes data to validate inputs without actually writing files",
+				},
 			},
 		},
 	}

strategies/csv/csv.go

@@ -54,6 +54,10 @@ func Import(c strategies.Context) error {
 	// from the MongoDB export.
 	input := c.String("input")
 
+	// dryRun indicates that the strategy should not write files and is used for
+	// validation.
+	dryRun := c.Bool("dryRun")
+
 	// auth is the identifier for the type of authentication profiles to be
 	// created for the users.
 	auth := c.String("auth")
@@ -189,7 +193,7 @@ func Import(c strategies.Context) error {
 	startedCommentsAt := time.Now()
 	logrus.Debug("processing comments")
 
-	commentsWriter, err := utility.NewJSONWriter(commentsOutputFileName)
+	commentsWriter, err := utility.NewJSONWriter(dryRun, commentsOutputFileName)
 	if err != nil {
 		return errors.Wrap(err, "could not create comment writer")
 	}
@@ -296,7 +300,7 @@ func Import(c strategies.Context) error {
 	startedUsersAt := time.Now()
 	logrus.Debug("processing users")
 
-	usersWriter, err := utility.NewJSONWriter(usersOutputFileName)
+	usersWriter, err := utility.NewJSONWriter(dryRun, usersOutputFileName)
 	if err != nil {
 		return errors.Wrap(err, "could not create users writer")
 	}
@@ -414,7 +418,7 @@ func Import(c strategies.Context) error {
 	startedStoriesAt := time.Now()
 	logrus.Debug("processing stories")
 
-	storiesWriter, err := utility.NewJSONWriter(storiesOutputFileName)
+	storiesWriter, err := utility.NewJSONWriter(dryRun, storiesOutputFileName)
 	if err != nil {
 		return errors.Wrap(err, "could not create story writer")
 	}

strategies/legacy/context.go

@@ -6,6 +6,7 @@ import (
 	"github.com/coralproject/coral-importer/common"
 	"github.com/coralproject/coral-importer/common/coral"
 	"github.com/coralproject/coral-importer/strategies"
+	"github.com/sirupsen/logrus"
 )
 
 type CommentRef struct {
@@ -41,7 +42,16 @@ func NewContext(c strategies.Context) *Context {
 	// from the MongoDB export.
 	input := c.String("input")
 
+	// dryRun indicates that the strategy should not write files and is used for
+	// validation.
+	dryRun := c.Bool("dryRun")
+
+	if dryRun {
+		logrus.Warn("dry run is enabled, files will not be written")
+	}
+
 	return &Context{
+		DryRun:   dryRun,
 		TenantID: tenantID,
 		SiteID:   siteID,
 		Filenames: Filenames{
@@ -85,6 +95,7 @@ type Filenames struct {
 }
 
 type Context struct {
+	DryRun        bool
 	TenantID      string
 	SiteID        string
 	Filenames     Filenames

strategies/legacy/legacy.go

@@ -133,13 +133,13 @@ func SeedCommentsReferences(ctx *Context) error {
 }
 
 func WriteCommentActions(ctx *Context) error {
-	commentActionsWriter, err := utility.NewJSONWriter(ctx.Filenames.Output.CommentActions)
+	commentActionsWriter, err := utility.NewJSONWriter(ctx.DryRun, ctx.Filenames.Output.CommentActions)
 	if err != nil {
 		return errors.Wrap(err, "could not create commentActionsWriter")
 	}
 	defer commentActionsWriter.Close()
 
-	bar, err := utility.NewLineCounter("Writing Comment Actions", ctx.Filenames.Input.Actions)
+	bar, err := utility.NewLineCounter("Comment Actions", ctx.Filenames.Input.Actions)
 	if err != nil {
 		return errors.Wrap(err, "could not count actions file")
 	}
@@ -199,13 +199,13 @@ func WriteCommentActions(ctx *Context) error {
 }
 
 func WriteComments(ctx *Context) error {
-	commentsWriter, err := utility.NewJSONWriter(ctx.Filenames.Output.Comments)
+	commentsWriter, err := utility.NewJSONWriter(ctx.DryRun, ctx.Filenames.Output.Comments)
 	if err != nil {
 		return errors.Wrap(err, "could not create comments writer")
 	}
 	defer commentsWriter.Close()
 
-	bar, err := utility.NewLineCounter("Writing Comments", ctx.Filenames.Input.Comments)
+	bar, err := utility.NewLineCounter("Comments", ctx.Filenames.Input.Comments)
 	if err != nil {
 		return errors.Wrap(err, "could not count comments file")
 	}
@@ -266,13 +266,13 @@ func WriteComments(ctx *Context) error {
 }
 
 func WriteStories(ctx *Context) error {
-	storiesWriter, err := utility.NewJSONWriter(ctx.Filenames.Output.Stories)
+	storiesWriter, err := utility.NewJSONWriter(ctx.DryRun, ctx.Filenames.Output.Stories)
 	if err != nil {
 		return errors.Wrap(err, "could not create stories writer")
 	}
 	defer storiesWriter.Close()
 
-	bar, err := utility.NewLineCounter("Writing Stories", ctx.Filenames.Input.Assets)
+	bar, err := utility.NewLineCounter("Stories", ctx.Filenames.Input.Assets)
 	if err != nil {
 		return errors.Wrap(err, "could not count assets file")
 	}
@@ -322,13 +322,13 @@ func WriteStories(ctx *Context) error {
 }
 
 func WriteUsers(ctx *Context) error {
-	usersWriter, err := utility.NewJSONWriter(ctx.Filenames.Output.Users)
+	usersWriter, err := utility.NewJSONWriter(ctx.DryRun, ctx.Filenames.Output.Users)
 	if err != nil {
 		return errors.Wrap(err, "could not create users writer")
 	}
 	defer usersWriter.Close()
 
-	bar, err := utility.NewLineCounter("Writing Users", ctx.Filenames.Input.Users)
+	bar, err := utility.NewLineCounter("Users", ctx.Filenames.Input.Users)
 	if err != nil {
 		return errors.Wrap(err, "could not count users file")
 	}