ghokin: manage non UTF-8 charset detection

* Add an automatic conversion to UTF-8 charset to be able to manage formatting * Add a warning to the README about how non UTF-8 files are managed
antham · Dec 29, 2023 · af66b75 · af66b75
1 parent 94e710f
commit af66b75
Show file tree

Hide file tree

Showing 7 changed files with 56 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -37,6 +37,8 @@ Flags:
 Use "ghokin [command] --help" for more information about a command.
 ```
 
+⚠️ Ghokin works only on `UTF-8` encoded files, it will detect and convert automatically files that are not encoded in this charset.
+
 ### fmt stdout
 
 Dump stdin or a feature file formatted on stdout

diff --git a/ghokin/file_manager.go b/ghokin/file_manager.go
@@ -3,12 +3,15 @@ package ghokin
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	mpath "path"
 	"path/filepath"
 	"sync"
 
 	"github.com/antham/ghokin/v3/ghokin/internal/transformer"
+	"github.com/saintfish/chardet"
+	"golang.org/x/net/html/charset"
 )
 
 // ProcessFileError is emitted when processing a file trigger an error
@@ -45,6 +48,21 @@ func (f FileManager) Transform(filename string) ([]byte, error) {
 	if err != nil {
 		return []byte{}, err
 	}
+	detector := chardet.NewTextDetector()
+	result, err := detector.DetectBest(content)
+	if err != nil {
+		return []byte{}, err
+	}
+	if result.Charset != "UTF-8" {
+		r, err := charset.NewReaderLabel(result.Charset, bytes.NewBuffer(content))
+		if err != nil {
+			return []byte{}, err
+		}
+		content, err = io.ReadAll(r)
+		if err != nil {
+			return []byte{}, err
+		}
+	}
 	contentTransformer := &transformer.ContentTransformer{}
 	contentTransformer.DetectSettings(content)
 	content = contentTransformer.Prepare(content)

diff --git a/ghokin/file_manager_test.go b/ghokin/file_manager_test.go
@@ -47,6 +47,15 @@ func TestFileManagerTransform(t *testing.T) {
 				assert.EqualValues(t, string(b), string(buf))
 			},
 		},
+		{
+			"fixtures/iso-8859-1-encoding.input.feature",
+			func(buf []byte, err error) {
+				assert.NoError(t, err)
+				b, e := os.ReadFile("fixtures/iso-8859-1-encoding.expected.feature")
+				assert.NoError(t, e)
+				assert.EqualValues(t, string(b), string(buf))
+			},
+		},
 		{
 			"fixtures/",
 			func(buf []byte, err error) {

diff --git a/ghokin/fixtures/iso-8859-1-encoding.expected.feature b/ghokin/fixtures/iso-8859-1-encoding.expected.feature
@@ -0,0 +1,11 @@
+Feature: aäáeéoóouüußä
+  In order to test something
+  As a felhasználó
+  I want to be able to do something
+
+  Scenario: Test
+    Given those values
+      | value         |
+      | aäáeéoóouüußä |
+    When I do 'aäáeéoóouüußä'
+    Then aäáeéoóouüußä happens
diff --git a/ghokin/fixtures/iso-8859-1-encoding.input.feature b/ghokin/fixtures/iso-8859-1-encoding.input.feature
@@ -0,0 +1,11 @@
+Feature: aäáeéoóouüußä
+      In order to test something
+      As a felhasználó
+      I want to be able to do something
+
+  Scenario: Test
+    Given those values
+      | value                       |
+      | aäáeéoóouüußä               |
+    When I do 'aäáeéoóouüußä'
+    Then aäáeéoóouüußä happens
diff --git a/go.mod b/go.mod
@@ -5,9 +5,11 @@ require (
 	github.com/fatih/color v1.16.0
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.8.4
+	golang.org/x/net v0.19.0
 )
 
 go 1.13
diff --git a/go.sum b/go.sum
@@ -1567,6 +1567,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
+github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -1802,6 +1804,7 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=