Merge pull request #321 from ikawaha/develop

Release candidate

.github/stalebot.yml

@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 60
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 14
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/go.yml

@@ -28,6 +28,13 @@ jobs:
           go-version-file: 'go.mod'
           cache: true
 
+      - name: Remove symlink 2to3
+        if: matrix.os == 'macos-latest'
+        run: |
+          : # Workaround GitHub Actions Python issues
+          : # https://github.com/Homebrew/homebrew-core/issues/165793#issuecomment-1989441193
+          brew unlink python && brew link --overwrite python
+
       - name: Set up Graphviz
         uses: ts-graphviz/setup-graphviz@v2
 

.github/workflows/release.yml

@@ -38,6 +38,14 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      # Login to GitHub Container Registry before push.
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GO_RELEASER_TOKEN }}
+
       # Build and release
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v5

.goreleaser.yml

@@ -70,6 +70,7 @@ brews:
 dockers:
   - image_templates:
       - "ikawaha/kagome:{{ .Version }}-amd64"
+      - "ghcr.io/ikawaha/kagome:{{ .Version }}-amd64"
     use: buildx
     goarch: amd64
     dockerfile: Dockerfile
@@ -78,6 +79,7 @@ dockers:
 
   - image_templates:
       - "ikawaha/kagome:{{ .Version }}-arm64"
+      - "ghcr.io/ikawaha/kagome:{{ .Version }}-arm64"
     use: buildx
     goarch: arm64
     dockerfile: Dockerfile
@@ -86,6 +88,7 @@ dockers:
 
   - image_templates:
       - "ikawaha/kagome:{{ .Version }}-arm32v5"
+      - "ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v5"
     use: buildx
     goarch: arm
     goarm: 5
@@ -95,6 +98,7 @@ dockers:
 
   - image_templates:
       - "ikawaha/kagome:{{ .Version }}-arm32v6"
+      - "ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v6"
     use: buildx
     goarch: arm
     goarm: 6
@@ -104,6 +108,7 @@ dockers:
 
   - image_templates:
       - "ikawaha/kagome:{{ .Version }}-arm32v7"
+      - "ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v7"
     use: buildx
     goarch: arm
     goarm: 7
@@ -114,6 +119,7 @@ dockers:
 # Create multiarch manifest file of Docker image. This will bind all the images
 # above into a single manifest file for "latest" and each version tag.
 docker_manifests:
+  # Manifest for Docker Hub
   - name_template: ikawaha/kagome:{{ .Version }}
     image_templates:
       - ikawaha/kagome:{{ .Version }}-amd64
@@ -127,4 +133,20 @@ docker_manifests:
       - ikawaha/kagome:{{ .Version }}-arm64
       - ikawaha/kagome:{{ .Version }}-arm32v5
       - ikawaha/kagome:{{ .Version }}-arm32v6
-      - ikawaha/kagome:{{ .Version }}-arm32v7
+      - ikawaha/kagome:{{ .Version }}-arm32v7
+
+  # Manifest for GitHub Container Registry
+  - name_template: ghcr.io/ikawaha/kagome:{{ .Version }}
+    image_templates:
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-amd64
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-arm64
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v5
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v6
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v7
+  - name_template: ghcr.io/ikawaha/kagome:latest
+    image_templates:
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-amd64
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-arm64
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v5
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v6
+      - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v7

README.md

@@ -276,16 +276,25 @@ A debug tool of tokenize process outputs a lattice in graphviz dot format.
 ```sh
 # Compatible architectures: AMD64, Arm64, Arm32 (Arm v5, v6 and v7)
 docker pull ikawaha/kagome:latest
+
+# Alternatively, you can pull from GitHub Container Registry
+docker pull ghcr.io/ikawaha/kagome:latest
 ```
 
 ```sh
 # Interactive/REPL mode
 docker run --rm -it ikawaha/kagome:latest
+
+# If pulling from GitHub Container Registry
+docker run --rm -it ghcr.io/ikawaha/kagome:latest
 ```
 
 ```sh
 # Server mode (http://localhost:6060)
 docker run --rm -p 6060:6060 ikawaha/kagome:latest server
+
+# If pulling from GitHub Container Registry
+docker run --rm -p 6060:6060 ghcr.io/ikawaha/kagome:latest server
 ```
 
 # Building to WebAssembly

_examples/db_search/README.md

@@ -0,0 +1,71 @@
+# Full-text search with Kagome and SQLite3
+
+This example provides a practical example of how to work with Japanese text data and **perform efficient [full-text search](https://en.wikipedia.org/wiki/Full-text_search) using Kagome and SQLite3**.
+
+- Target text data is as follows:
+
+```text
+人魚は、南の方の海にばかり棲んでいるのではありません。
+北の海にも棲んでいたのであります。
+北方の海の色は、青うございました。
+ある時、岩の上に、女の人魚があがって、
+あたりの景色を眺めながら休んでいました。
+小川未明 『赤い蝋燭と人魚』
+```
+
+- Example output:
+
+```shellsession
+$ cd /path/to/kagome/_examples/db_search
+$ go run .
+Searching for: 人魚
+  Found content: 人魚は、南の方の海にばかり棲んでいるのではありません。 at line: 1
+  Found content: ある時、岩の上に、女の人魚があがって、 at line: 4
+  Found content: 小川未明 『赤い蝋燭と人魚』 at line: 6
+Searching for: 人
+  No results found
+Searching for: 北方
+  Found content: 北方の海の色は、青うございました。 at line: 3
+Searching for: 北
+  Found content: 北の海にも棲んでいたのであります。 at line: 2
+```
+
+- [View main.go](main.go)
+
+## Details
+
+In this example, each line of text is inserted into a row of the SQLite3 database, and then the database is searched for the word "人魚", "人", "北方" and "北".
+
+When inserting text data into the database, Kagome is used to tokenize the text into words.
+
+The string (or a line) tokenized by Kagome, a.k.a. "Wakati", is recorded in a separate table for [FTS4](https://www.sqlite.org/fts3.html) (Full-Text-Search) relative to the original text.
+
+This allows Unicode text data that is not separated by spaces, such as Japanese, to be searched by FTS.
+
+Note that it is searching by word and not by character. For example "人" doesn't match "人魚". Likewise, "北" doesn't match "北方".
+
+This is due to the fact that the FTS4 module in SQLite3 is designed to search for words, not characters.
+
+### Aim of this example
+
+This example can be useful in scenarios where you need to perform full-text searches on Japanese text.
+
+It demonstrates how to tokenize Japanese text using Kagome, which is a common requirement when working with text data in the Japanese language.
+
+By using SQLite with FTS4, it efficiently manages and searches through a large amount of text data, making it suitable for applications like:
+
+1. **Search Engines:** You can use this code as a basis for building a search engine that indexes and searches Japanese text content.
+2. **Document Management Systems:** This code can be integrated into a document management system to enable full-text search capabilities for Japanese documents.
+3. **Content Recommendation Systems:** When you have a large collection of Japanese content, you can use this code to implement content recommendation systems based on user queries.
+4. **Chatbots and NLP:**  If you're building chatbots or natural language processing (NLP) systems for Japanese language, this code can assist in text analysis and search within the chatbot's knowledge base.
+
+## Acknowledgements
+
+This example is taken in part from the following book for reference.
+
+- p.204, 9.2 "データーベース登録プログラム", "Go言語プログラミングエッセンス エンジニア選書"
+  - Written by: [Mattn](https://github.com/mattn)
+  - Published: 2023/3/9 (技術評論社)
+  - ISBN: 4297134195 / 978-4297134198
+  - ASIN: B0BVZCJQ4F / [https://amazon.co.jp/dp/4297134195](https://amazon.co.jp/dp/4297134195)
+  - Original sample code: [https://github.com/mattn/aozora-search](https://github.com/mattn/aozora-search)

sample/_example/go.mod → _examples/db_search/go.mod

@@ -1,4 +1,4 @@
-module kagome/examples
+module kagome/examples/db_search
 
 go 1.19
 

sample/_example/db_search/main.go → _examples/db_search/main.go

@@ -1,35 +1,9 @@
 /*
-# TL; DR
+# Full-text search with Kagome and SQLite3
 
 This example provides a practical example of how to work with Japanese text data and perform efficient full-text search using Kagome and SQLite3.
 
-# TS; WM
-
-In this example, each line of text is inserted into a row of the SQLite3 database, and then the database is searched for the word "人魚" and "人".
-
-Note that the string tokenized by Kagome, a.k.a. "Wakati", is recorded in a separate table for FTS (Full-Text-Search) at the same time as the original text.
-
-This allows Unicode text data that is not separated by spaces, such as Japanese, to be searched by FTS.
-
-Aim of this example:
-
-This example can be useful in scenarios where you need to perform full-text searches on Japanese text. It demonstrates how to tokenize Japanese text using Kagome, which is a common requirement when working with text data in the Japanese language. By using SQLite with FTS4, it efficiently manages and searches through a large amount of text data, making it suitable for applications like:
-
-1. **Search Engines:** You can use this code as a basis for building a search engine that indexes and searches Japanese text content.
-2. **Document Management Systems:**	This code can be integrated into a document management system to enable full-text search capabilities for Japanese documents.
-3. **Content Recommendation Systems:** When you have a large collection of Japanese content, you can use this code to implement content recommendation systems based on user queries.
-4. **Chatbots and NLP:**  If you're building chatbots or natural language processing (NLP) systems for Japanese language, this code can assist in text analysis and search within the chatbot's knowledge base.
-
-Acknowledgements:
-
-This example is taken in part from the following book for reference.
-
-- p.204, 9.2 "データーベース登録プログラム", "Go言語プログラミングエッセンス エンジニア選書"
-  - Written by: Mattn
-  - Published: 2023/3/9 (技術評論社)
-  - ISBN: 4297134195 / 978-4297134198
-  - ASIN: B0BVZCJQ4F / https://amazon.co.jp/dp/4297134195
-  - Original sample code: https://github.com/mattn/aozora-search
+For details and acknowledgements, see the README.md file in the same directory.
 */
 package main
 
@@ -39,6 +13,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"slices"
 	"strings"
 
 	"github.com/ikawaha/kagome-dict/ipa"
@@ -165,6 +140,9 @@ func insertSearchToken(db *sql.DB, rowID int64, content string) error {
 	}
 
 	seg := tknzr.Wakati(content)
+
+	seg = slices.Compact(seg) // remove duplicate segment tokens
+
 	tokenizedContent := strings.Join(seg, " ")
 
 	_, err = db.Exec(

_examples/go.work

@@ -0,0 +1,9 @@
+go 1.19
+
+use (
+	./db_search
+	./tokenize
+	./user_dict
+	./wakati
+	./wasm
+)

_examples/tokenize/README.md

@@ -0,0 +1,28 @@
+# Tokenizing Example with Kagome
+
+## Analyzing a Japanese text into words and parts of speech with Kagome
+
+This example demonstrates how to analyzes a sentence (tokenize) and get the part-of-speech (POS) of each word using Kagome.
+
+- Target text data is as follows:
+
+```text
+すもももももももものうち
+```
+
+- Example output:
+
+```shellsession
+$ cd /path/to/kagome/_examples/tokenize
+$ go run .
+---tokenize---
+すもも  名詞,一般,*,*,*,*,すもも,スモモ,スモモ
+も      助詞,係助詞,*,*,*,*,も,モ,モ
+もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+も      助詞,係助詞,*,*,*,*,も,モ,モ
+もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+の      助詞,連体化,*,*,*,*,の,ノ,ノ
+うち    名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
+```
+
+> __Note__ that tokenization varies depending on the dictionary used. In this example we use the IPA dictionary.

_examples/tokenize/go.mod

@@ -0,0 +1,12 @@
+module kagome/examples/tokenize
+
+go 1.19
+
+require (
+	github.com/ikawaha/kagome-dict/ipa v1.0.10
+	github.com/ikawaha/kagome/v2 v2.9.3
+)
+
+require github.com/ikawaha/kagome-dict v1.0.9 // indirect
+
+replace github.com/ikawaha/kagome/v2 => ../../

_examples/tokenize/go.sum

@@ -0,0 +1,4 @@
+github.com/ikawaha/kagome-dict v1.0.9 h1:1Gg735LbBYsdFu13fdTvW6eVt0qIf5+S2qXGJtlG8C0=
+github.com/ikawaha/kagome-dict v1.0.9/go.mod h1:mn9itZLkFb6Ixko7q8eZmUabHbg3i9EYewnhOtvd2RM=
+github.com/ikawaha/kagome-dict/ipa v1.0.10 h1:wk9I21yg+fKdL6HJB9WgGiyXIiu1VttumJwmIRwn0g8=
+github.com/ikawaha/kagome-dict/ipa v1.0.10/go.mod h1:rbaOKrF58zhtpV2+2sVZBj0sUSp9dVKPjr660MehJbs=

sample/_example/tokenize/main.go → _examples/tokenize/main.go

@@ -22,12 +22,12 @@ func main() {
 	}
 
 	// Output:
-	//---tokenize---
-	//すもも	名詞,一般,*,*,*,*,すもも,スモモ,スモモ
-	//も	助詞,係助詞,*,*,*,*,も,モ,モ
-	//もも	名詞,一般,*,*,*,*,もも,モモ,モモ
-	//も	助詞,係助詞,*,*,*,*,も,モ,モ
-	//もも	名詞,一般,*,*,*,*,もも,モモ,モモ
-	//の	助詞,連体化,*,*,*,*,の,ノ,ノ
-	//うち	名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
+	// ---tokenize---
+	// すもも	名詞,一般,*,*,*,*,すもも,スモモ,スモモ
+	// も	助詞,係助詞,*,*,*,*,も,モ,モ
+	// もも	名詞,一般,*,*,*,*,もも,モモ,モモ
+	// も	助詞,係助詞,*,*,*,*,も,モ,モ
+	// もも	名詞,一般,*,*,*,*,もも,モモ,モモ
+	// の	助詞,連体化,*,*,*,*,の,ノ,ノ
+	// うち	名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
 }

_examples/user_dict/go.mod

@@ -0,0 +1,11 @@
+module kagome/examples/user_dict
+
+go 1.19
+
+require (
+	github.com/ikawaha/kagome-dict v1.0.9
+	github.com/ikawaha/kagome-dict/ipa v1.0.10
+	github.com/ikawaha/kagome/v2 v2.9.3
+)
+
+replace github.com/ikawaha/kagome/v2 => ../../

_examples/user_dict/go.sum

@@ -0,0 +1,4 @@
+github.com/ikawaha/kagome-dict v1.0.9 h1:1Gg735LbBYsdFu13fdTvW6eVt0qIf5+S2qXGJtlG8C0=
+github.com/ikawaha/kagome-dict v1.0.9/go.mod h1:mn9itZLkFb6Ixko7q8eZmUabHbg3i9EYewnhOtvd2RM=
+github.com/ikawaha/kagome-dict/ipa v1.0.10 h1:wk9I21yg+fKdL6HJB9WgGiyXIiu1VttumJwmIRwn0g8=
+github.com/ikawaha/kagome-dict/ipa v1.0.10/go.mod h1:rbaOKrF58zhtpV2+2sVZBj0sUSp9dVKPjr660MehJbs=