impliment find similar tags

README.md

@@ -1,33 +1,5 @@
-# go-project-boilerplate
+# hugo-utilities
 
-[![go](https://github.com/sters/go-project-boilerplate/workflows/Go/badge.svg)](https://github.com/sters/go-project-boilerplate/actions?query=workflow%3AGo)
-[![codecov](https://codecov.io/gh/sters/go-project-boilerplate/branch/main/graph/badge.svg)](https://codecov.io/gh/sters/go-project-boilerplate)
-[![go-report](https://goreportcard.com/badge/github.com/sters/go-project-boilerplate)](https://goreportcard.com/report/github.com/sters/go-project-boilerplate)
-
-My go project boilerplate.
-
-## Includes
-
-- Makefile
-  - run
-  - test
-  - cover
-  - Tools install from `./tools/tools.go`
-- Github Actions
-  - Go
-    - Lint by golangcilint
-    - Run test and upload test coverage to codecov
-  - Release
-    - Make release when vX.X.X tag is added by goreleaser.
-- README
-  - Badge: Github Actions/Go
-  - Badge: Codecov
-  - Badge: Go Report
-
-## TODO when use this
-
-- [ ] Change `cmd/xxxxx` directory name
-- [ ] Change run task in `Makefile`
-- [ ] Change package name in `go.mod`
-- [ ] Change main in `.goreleaser.yml`
-- [ ] Update `README.md`
+[![go](https://github.com/sters/hugo-utilities/workflows/Go/badge.svg)](https://github.com/sters/hugo-utilities/actions?query=workflow%3AGo)
+[![codecov](https://codecov.io/gh/sters/hugo-utilities/branch/main/graph/badge.svg)](https://codecov.io/gh/sters/hugo-utilities)
+[![go-report](https://goreportcard.com/badge/github.com/sters/hugo-utilities)](https://goreportcard.com/report/github.com/sters/hugo-utilities)

cmd/find-similar-tags/main.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/agnivade/levenshtein"
+	"github.com/sters/hugo-utilities/util"
+	hugocontent "github.com/sters/simple-hugo-content-parse"
+)
+
+type similarTagRelation struct {
+	a        string
+	b        string
+	distance int
+}
+
+type similarTagRelationHash string
+
+func (s *similarTagRelation) Hash() similarTagRelationHash {
+	return similarTagRelationHash(s.a + "-" + s.b)
+}
+
+func (s *similarTagRelation) HashRev() similarTagRelationHash {
+	return similarTagRelationHash(s.b + "-" + s.a)
+}
+
+type similarTagRelations map[similarTagRelationHash]*similarTagRelation
+
+func (s similarTagRelations) appendWithoutDuplecation(r *similarTagRelation) {
+	if _, ok := s[r.Hash()]; ok {
+		return
+	}
+	if _, ok := s[r.HashRev()]; ok {
+		return
+	}
+	s[r.Hash()] = r
+}
+
+func getAllTags(contents []*hugocontent.MarkdownContent, field string) []string {
+	tags := make([]string, 0, len(contents))
+	check := map[string]struct{}{}
+	for _, c := range contents {
+		for _, t := range c.FrontMatter.GetStrings(field) {
+			if _, ok := check[t]; ok {
+				continue
+			}
+
+			tags = append(tags, t)
+			check[t] = struct{}{}
+		}
+	}
+
+	return tags
+}
+
+func main() {
+	t := flag.Float64("threshold", 0.7, "")
+	ff := flag.String("field", "tags", "")
+	basicFlags := util.ParseFlag()
+	flags := struct {
+		threshold float64
+		field     string
+	}{
+		threshold: *t,
+		field:     *ff,
+	}
+
+	contents, err := util.ReadAllContents(basicFlags.BaseDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed: %s", err)
+		os.Exit(1)
+	}
+
+	tags := getAllTags(contents, flags.field)
+	r := similarTagRelations{}
+
+	for i, tagA := range tags {
+		for j, tagB := range tags {
+			if i >= j || tagA == tagB {
+				continue
+			}
+
+			distance := levenshtein.ComputeDistance(tagA, tagB)
+			g := func(s string) int {
+				return int(flags.threshold * float64(len([]rune(s))))
+			}
+			if distance >= g(tagA) || distance >= g(tagB) {
+				continue
+			}
+
+			r.appendWithoutDuplecation(
+				&similarTagRelation{
+					a:        tagA,
+					b:        tagB,
+					distance: distance,
+				},
+			)
+		}
+	}
+
+	result := make([]string, 0, len(r))
+	for _, n := range r {
+		result = append(
+			result,
+			fmt.Sprintf("%s, %s: distance = %d", n.a, n.b, n.distance),
+		)
+	}
+
+	sort.Slice(result, func(i, j int) bool { return result[i] < result[j] })
+
+	fmt.Printf(strings.Join(result, "\n"))
+}

go.mod

@@ -1,8 +1,16 @@
-module github.com/sters/go-project-boilerplate
+module github.com/sters/hugo-utilities
 
 go 1.16
 
 require (
+	github.com/agnivade/levenshtein v1.1.1
+	github.com/fatih/color v1.12.0 // indirect
+	github.com/goccy/go-yaml v1.8.9 // indirect
 	github.com/golangci/golangci-lint v1.40.1
-	github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c
+	github.com/mattn/go-isatty v0.0.13 // indirect
+	github.com/morikuni/failure v0.14.0
+	github.com/quasilyte/go-consistent v0.0.0-20200404105227-766526bf1e96
+	github.com/sters/simple-hugo-content-parse v0.1.0
+	github.com/stretchr/testify v1.7.0
+	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
 )