feat: support markdown to text function in text operator

go.mod

@@ -38,6 +38,7 @@ require (
 	google.golang.org/api v0.172.0
 	google.golang.org/grpc v1.62.1
 	google.golang.org/protobuf v1.33.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -123,5 +124,4 @@ require (
 	google.golang.org/genproto v0.0.0-20240221002015-b0ce06bbee7c // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

operator/text/v0/convert.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"fmt"
 	"strings"
+	"time"
 
 	"encoding/base64"
 
 	"code.sajari.com/docconv"
+	"gopkg.in/yaml.v3"
 
 	"github.com/instill-ai/component/base"
 )
@@ -43,17 +45,13 @@ func getContentTypeFromBase64(base64String string) (string, error) {
 	return parts[0], nil
 }
 
-func convertToText(input ConvertToTextInput) (ConvertToTextOutput, error) {
+type converter interface {
+	convert(contentType string, b []byte) (ConvertToTextOutput, error)
+}
 
-	contentType, err := getContentTypeFromBase64(input.Doc)
-	if err != nil {
-		return ConvertToTextOutput{}, err
-	}
+type docconvConverter struct{}
 
-	b, err := base64.StdEncoding.DecodeString(base.TrimBase64Mime(input.Doc))
-	if err != nil {
-		return ConvertToTextOutput{}, err
-	}
+func (d docconvConverter) convert(contentType string, b []byte) (ConvertToTextOutput, error) {
 
 	res, err := docconv.Convert(bytes.NewReader(b), contentType, false)
 	if err != nil {
@@ -71,3 +69,75 @@ func convertToText(input ConvertToTextInput) (ConvertToTextOutput, error) {
 		Error: res.Error,
 	}, nil
 }
+
+type markdownConverter struct{}
+
+func extractMetadata(content string) map[string]string {
+	var metadata map[string]string
+
+	if !strings.HasPrefix(content, "---") {
+		return metadata
+	}
+
+	parts := strings.SplitN(content, "---", 3)
+	if len(parts) < 3 {
+		return metadata
+	}
+
+	yamlContent := parts[1]
+	err := yaml.Unmarshal([]byte(yamlContent), &metadata)
+	if err != nil {
+		return metadata
+	}
+
+	return metadata
+}
+
+func (m markdownConverter) convert(contentType string, b []byte) (ConvertToTextOutput, error) {
+
+	before := time.Now()
+	content := string(b)
+	metadata := extractMetadata(content)
+
+	duration := time.Since(before)
+	millis := duration.Milliseconds()
+
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+
+	return ConvertToTextOutput{
+		Body:  content,
+		Meta:  metadata,
+		MSecs: uint32(millis),
+		Error: "",
+	}, nil
+}
+
+func convertToText(input ConvertToTextInput) (ConvertToTextOutput, error) {
+
+	contentType, err := getContentTypeFromBase64(input.Doc)
+	if err != nil {
+		return ConvertToTextOutput{}, err
+	}
+
+	b, err := base64.StdEncoding.DecodeString(base.TrimBase64Mime(input.Doc))
+	if err != nil {
+		return ConvertToTextOutput{}, err
+	}
+
+	var converter converter
+	switch contentType {
+	case "application/octet-stream":
+		converter = markdownConverter{}
+	default:
+		converter = docconvConverter{}
+	}
+
+	res, err := converter.convert(contentType, b)
+	if err != nil {
+		return ConvertToTextOutput{}, err
+	}
+
+	return res, nil
+}

operator/text/v0/convert_test.go

@@ -55,6 +55,10 @@ func TestConvertToText(t *testing.T) {
 			name:     "Convert txt file",
 			filepath: "testdata/test.txt",
 		},
+		{
+			name:     "Convert md file",
+			filepath: "testdata/test.md",
+		},
 	}
 
 	for _, test := range tests {

operator/text/v0/testdata/test.md

@@ -0,0 +1,8 @@
+---
+title: "Sample Markdown Document"
+author: "John Doe"
+date: "2023-05-31"
+description: "This is a sample Markdown document with YAML front matter."
+---
+# Introduction
+This is a sample Markdown file to demonstrate extracting metadata.