Add remove marker function (#56)

* Add remove marker function

* Use marker flag as a function argument for easier test

internal/cli/generate.go

@@ -24,11 +24,13 @@ This approach allows the input file to be full of Importer markes without actual
 		Args: cobra.MinimumNArgs(1),
 		RunE: executeGenerate,
 	}
-	generateTargetFile string
+	generateTargetFile  string
+	generateKeepMarkers bool
 )
 
 func init() {
 	generateCliCmd.Flags().StringVarP(&generateTargetFile, "out", "o", "", "write to `FILE`")
+	generateCliCmd.Flags().BoolVar(&generateKeepMarkers, "keep-markers", false, "keep Importer Markers from the generated result")
 }
 
 func executeGenerate(cmd *cobra.Command, args []string) error {
@@ -43,14 +45,15 @@ func executeGenerate(cmd *cobra.Command, args []string) error {
 
 	arg := args[0]
 	out := generateTargetFile
-	if err := generate(arg, out); err != nil {
+	keepMarkers := generateKeepMarkers
+	if err := generate(arg, out, keepMarkers); err != nil {
 		return fmt.Errorf("failed to generate for '%s', %v", arg, err)
 	}
 
 	return nil
 }
 
-func generate(fileName string, targetFilepath string) error {
+func generate(fileName string, targetFilepath string, keepMarkers bool) error {
 	f, err := os.Open(fileName)
 	if err != nil {
 		return err
@@ -67,6 +70,10 @@ func generate(fileName string, targetFilepath string) error {
 		return err
 	}
 
+	if !keepMarkers {
+		file.RemoveMarkers()
+	}
+
 	if targetFilepath != "" {
 		return file.WriteAfterTo(targetFilepath)
 	}

internal/cli/generate_test.go

@@ -19,31 +19,37 @@ var updateGolden = flag.Bool("update", false, "update golden files")
 func TestGenerateStdout(t *testing.T) {
 	cases := map[string]struct {
 		// Input
-		inputFile string
+		inputFile   string
+		keepMarkers bool
 
 		// Output
 		wantFile      string
 		wantErrString string
 	}{
 		"markdown": {
-			inputFile: "../../testdata/markdown/simple-before.md",
-			wantFile:  "../../testdata/markdown/simple-updated.md",
+			inputFile:   "../../testdata/markdown/simple-before.md",
+			keepMarkers: true,
+			wantFile:    "../../testdata/markdown/simple-updated.md",
 		},
 		"markdown with exporter": {
-			inputFile: "../../testdata/markdown/import-with-exporter-before.md",
-			wantFile:  "../../testdata/markdown/import-with-exporter-updated.md",
+			inputFile:   "../../testdata/markdown/import-with-exporter-before.md",
+			keepMarkers: true,
+			wantFile:    "../../testdata/markdown/import-with-exporter-updated.md",
 		},
 		"yaml with exporter": {
-			inputFile: "../../testdata/yaml/demo-before.yaml",
-			wantFile:  "../../testdata/yaml/demo-updated.yaml",
+			inputFile:   "../../testdata/yaml/demo-before.yaml",
+			keepMarkers: true,
+			wantFile:    "../../testdata/yaml/demo-updated.yaml",
 		},
 		"yaml with exporter and align": {
-			inputFile: "../../testdata/yaml/align-with-exporter-before.yaml",
-			wantFile:  "../../testdata/yaml/align-with-exporter-updated.yaml",
+			inputFile:   "../../testdata/yaml/align-with-exporter-before.yaml",
+			keepMarkers: true,
+			wantFile:    "../../testdata/yaml/align-with-exporter-updated.yaml",
 		},
 		"yaml with exporter and align, k8s": {
-			inputFile: "../../testdata/yaml/k8s-color-svc-before.yaml",
-			wantFile:  "../../testdata/yaml/k8s-color-svc-updated.yaml",
+			inputFile:   "../../testdata/yaml/k8s-color-svc-before.yaml",
+			keepMarkers: true,
+			wantFile:    "../../testdata/yaml/k8s-color-svc-updated.yaml",
 		},
 		"error case: file not found": {
 			inputFile:     "does_not_exist",
@@ -60,7 +66,7 @@ func TestGenerateStdout(t *testing.T) {
 			fakeStdout := stdout.New(t)
 			defer fakeStdout.Close()
 
-			err := generate(tc.inputFile, "") // Empty second argument means generate writes to stdout
+			err := generate(tc.inputFile, "", tc.keepMarkers) // Empty second argument means generate writes to stdout
 			if err != nil {
 				if !strings.Contains(err.Error(), tc.wantErrString) {
 					t.Fatalf("error with generate, %v", err)
@@ -83,18 +89,21 @@ func TestGenerateStdout(t *testing.T) {
 func TestGenerateToFile(t *testing.T) {
 	cases := map[string]struct {
 		// Input
-		inputFile string
+		inputFile   string
+		keepMarkers bool
 
 		// Output
 		wantFile string
 	}{
 		"markdown": {
-			inputFile: "../../testdata/markdown/simple-before.md",
-			wantFile:  "../../testdata/markdown/simple-updated.md",
+			inputFile:   "../../testdata/markdown/simple-before.md",
+			keepMarkers: true,
+			wantFile:    "../../testdata/markdown/simple-updated.md",
 		},
 		"markdown with exporter": {
-			inputFile: "../../testdata/markdown/import-with-exporter-before.md",
-			wantFile:  "../../testdata/markdown/import-with-exporter-updated.md",
+			inputFile:   "../../testdata/markdown/import-with-exporter-before.md",
+			keepMarkers: true,
+			wantFile:    "../../testdata/markdown/import-with-exporter-updated.md",
 		},
 	}
 
@@ -105,7 +114,7 @@ func TestGenerateToFile(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = generate(tc.inputFile, tempFile.Name()) // Second argument for target file
+			err = generate(tc.inputFile, tempFile.Name(), tc.keepMarkers) // Second argument for target file
 			if err != nil {
 				t.Fatalf("error with generate, %v", err)
 			}

internal/file/process.go

@@ -1,7 +1,13 @@
 package file
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
+	"path/filepath"
+	"regexp"
+
+	"github.com/upsidr/importer/internal/marker"
 )
 
 const br = byte('\n')
@@ -35,3 +41,57 @@ func (f *File) ProcessMarkers() error {
 	f.ContentAfter = result
 	return nil
 }
+
+// RemoveMarkers removes Importer markers. This is useful for generated files
+// to have no marker input.
+func (f *File) RemoveMarkers() {
+	fileType := filepath.Ext(f.FileName)
+
+	var importerRe *regexp.Regexp
+	switch fileType {
+	case ".md":
+		importerRe = regexp.MustCompile(marker.ImporterMarkerMarkdown)
+	case ".yaml", ".yml":
+		importerRe = regexp.MustCompile(marker.ImporterMarkerYAML)
+	default:
+		// File that does not have supporting marker setup will be simply
+		// ignored.
+	}
+
+	var exporterRe *regexp.Regexp
+	switch fileType {
+	case ".md":
+		exporterRe = regexp.MustCompile(marker.ExporterMarkerMarkdown)
+	case ".yaml", ".yml":
+		exporterRe = regexp.MustCompile(marker.ExporterMarkerYAML)
+	default:
+		// File that does not have supporting marker setup will be simply
+		// ignored.
+	}
+
+	if importerRe == nil || exporterRe == nil {
+		return
+	}
+
+	newResult := []byte{}
+
+	scanner := bufio.NewScanner(bytes.NewReader(f.ContentAfter))
+	for scanner.Scan() {
+		currentLine := scanner.Bytes()
+		if s := importerRe.Find(currentLine); s != nil {
+			// Importer Marker found, ignore
+			continue
+		}
+		if s := exporterRe.Find(currentLine); s != nil {
+			// Exporter Marker found, ignore
+			continue
+		}
+
+		currentLine = append(currentLine, []byte("\n")...)
+		newResult = append(newResult, currentLine...)
+	}
+
+	f.ContentAfter = newResult
+
+	return
+}

internal/file/process_test.go

@@ -82,3 +82,129 @@ data
 		})
 	}
 }
+
+func TestRemoveMarkers(t *testing.T) {
+	cases := map[string]struct {
+		// Input
+		file *File
+
+		// Output
+		want []byte
+	}{
+		"markdown: without importer marker": {
+			file: &File{
+				FileName: "test-file.md",
+				ContentAfter: []byte(`
+A
+B
+C
+D
+E
+
+1
+2
+3
+4
+5
+`),
+			},
+			want: []byte(`
+A
+B
+C
+D
+E
+
+1
+2
+3
+4
+5
+`),
+		},
+		"markdown: with importer marker, should be removed": {
+			file: &File{
+				FileName: "test-file.md",
+				ContentAfter: []byte(`
+<!-- == importer: abc / begin from: abc.md#1~ == -->
+A
+B
+C
+D
+E
+<!-- == importer: abc / end == -->
+
+<!-- == export: bbb / begin == -->
+1
+2
+3
+4
+5
+<!-- == export: bbb / end == -->
+`),
+			},
+			want: []byte(`
+A
+B
+C
+D
+E
+
+1
+2
+3
+4
+5
+`),
+		},
+		"yaml: with importer marker, should be removed": {
+			file: &File{
+				FileName: "test-file.yaml",
+				ContentAfter: []byte(`
+# == i: abc / begin from: some-file.yaml#[abc] ==
+a:
+  b:
+    c: data
+# == i: abc / end ==
+`),
+			},
+			want: []byte(`
+a:
+  b:
+    c: data
+`),
+		},
+		"unknown file type: keep input as is": {
+			file: &File{
+				FileName: "test-file.dummy",
+				ContentAfter: []byte(`
+YAML like file, but not considered yaml due to the file extension.
+
+# == i: abc / begin from: some-file.yaml#[abc] ==
+a:
+  b:
+    c: data
+# == i: abc / end ==
+`),
+			},
+			want: []byte(`
+YAML like file, but not considered yaml due to the file extension.
+
+# == i: abc / begin from: some-file.yaml#[abc] ==
+a:
+  b:
+    c: data
+# == i: abc / end ==
+`),
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			tc.file.RemoveMarkers()
+			if diff := cmp.Diff(tc.want, tc.file.ContentAfter); diff != "" {
+				t.Errorf("parsed result didn't match (-want / +got)\n%s", diff)
+			}
+		})
+	}
+}

testdata/other/demo-generated.md

@@ -1,9 +1,7 @@
 <!-- == improter-generated-from: ../markdown/demo-before.md == -->
 # Markdown Demo
 
-<!-- == imptr: short-description / begin from: ./snippet-description.md#[for-demo] == -->
 This demonstrates how a markdown can import other file content.
 
 Importer is a CLI tool to read and process Importer and Exporter markers.  
 This can be easily integrated into CI/CD and automation setup.
-<!-- == imptr: short-description / end == -->

testdata/other/simple-generated.md

@@ -0,0 +1,13 @@
+<!-- == improter-generated-from: ../markdown/simple-before.md == -->
+# Simple Markdown Test
+
+"Lorem ipsum dolor sit amet,
+consectetur adipiscing elit,
+sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Ut enim ad minim veniam,
+quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
+Excepteur sint occaecat cupidatat non proident,
+sunt in culpa qui officia deserunt mollit anim id est laborum."
+
+Content after marker is left untouched.