Skip to content

Commit

Permalink
fix: 「...」の中での区切り文字の判定に句点のみが使われいた問題を修正 (#24)
Browse files Browse the repository at this point in the history 
センテンス分解に使うと区切り文字も「...」の中の区切り文字として扱うように追加した。

- "?", // question mark
- "!", //  exclamation mark
- "？", // (ja) zenkaku question mark
- "！" // (ja) zenkaku exclamation mark
  • Loading branch information
@azu
azu authored Feb 11, 2023
1 parent 08992d7 commit e3de7e0
Show file tree
Hide file tree
Showing 4 changed files with 77 additions and 35 deletions.
5 changes: 2 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,8 @@
"dependencies": {
"kuromojin": "^3.0.0",
"sentence-splitter": "^3.2.0",
"structured-source": "^3.0.2",
"textlint-rule-helper": "^2.0.0",
"textlint-util-to-string": "^3.1.1"
"textlint-rule-helper": "^2.3.0",
"textlint-util-to-string": "^3.3.0"
},
"prettier": {
"singleQuote": false,
Expand Down
24 changes: 16 additions & 8 deletions src/max-ten.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ function is括弧(token) {
}
return false;
}

/**
* 括弧などの記号的なTokenはスキップとして隣接するTokenを探す
* @see https://github.com/textlint-ja/textlint-rule-max-ten/issues/12
Expand Down Expand Up @@ -75,19 +76,20 @@ module.exports = function (context, options = {}) {
const kuten = options.kuten ?? defaultOptions.kuten;
const helper = new RuleHelper(context);
const { Syntax, RuleError, report, getSource } = context;
const separatorCharacters = [
"?", // question mark
"!", // exclamation mark
"？", // (ja) zenkaku question mark
"！" // (ja) zenkaku exclamation mark
].concat(kuten);
return {
[Syntax.Paragraph](node) {
if (helper.isChildNode(node, [Syntax.BlockQuote])) {
return;
}
const resultNode = splitAST(node, {
SeparatorParser: {
separatorCharacters: [
"?", // question mark
"!", // exclamation mark
"？", // (ja) zenkaku question mark
"！" // (ja) zenkaku exclamation mark
].concat(kuten)
separatorCharacters: separatorCharacters
}
});
const sentences = resultNode.children.filter((childNode) => childNode.type === SentenceSyntax.Sentence);
Expand All @@ -104,7 +106,13 @@ module.exports = function (context, options = {}) {
3. check tokens
*/
const checkSentence = async (sentence) => {
const source = new StringSource(sentence);
const source = new StringSource(sentence, {
replacer({ node, maskValue }) {
if (node.type === Syntax.Code) {
return maskValue("_");
}
}
});
const text = source.toString();
const tokens = await tokenize(text);
let currentTenCount = 0;
Expand Down Expand Up @@ -133,7 +141,7 @@ module.exports = function (context, options = {}) {
currentTenCount++;
lastToken = token;
}
if (surface === kuten) {
if (separatorCharacters.includes(surface)) {
// reset
currentTenCount = 0;
}
Expand Down
7 changes: 5 additions & 2 deletions test/max-ten-test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ tester.run("max-ten", rule, {
touten: "，",
kuten: "．"
}
}
},
// https://github.com/textlint-ja/textlint-rule-max-ten/issues/17
"紅ちゃんは、たたえなくなっていいって、思う？ もっと大事なものがあったら、スパイになれなくてもいいって、思う？",
"「紅ちゃんは、たたえなくなっていいって、思う？ もっと大事なものがあったら、スパイになれなくてもいいって、思う？」"
],
invalid: [
{
Expand Down Expand Up @@ -148,7 +151,7 @@ tester.run("max-ten", rule, {
{
// 3行目が、の問題
text: `このパラグラフはOKです。
変数の名前は、名前のルールが決まっていて、そのルールは複雑で、たくさんのルールがあるので、箇条書きしましょう。
3つめのパラグラフはもOKです。
Expand Down
76 changes: 54 additions & 22 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -913,7 +913,12 @@
readdirp "^2.2.1"
upath "^1.1.1"

"@textlint/ast-node-types@^4.2.1", "@textlint/ast-node-types@^4.2.4", "@textlint/ast-node-types@^4.4.2":
"@textlint/ast-node-types@^13.0.2", "@textlint/ast-node-types@^13.0.5":
version "13.2.0"
resolved "https://registry.yarnpkg.com/@textlint/ast-node-types/-/ast-node-types-13.2.0.tgz#9210439c88bc61664f8534a19cb7794dcc2061ed"
integrity sha512-P76rGK9SoN/f40yVW2Dep3QkXQOkAYTzEZjV0yBf/W9N0c81HUDJPS//aRbS3ml0Yb7TNgkXYm/ChTsL79RcAg==

"@textlint/ast-node-types@^4.4.2":
version "4.4.2"
resolved "https://registry.yarnpkg.com/@textlint/ast-node-types/-/ast-node-types-4.4.2.tgz#d3fda5c9086baba04bc75440039ccf18415c8446"
integrity sha512-m5brKbI7UY/Q8sbIZ7z1KB8ls04nRILshz5fPQ4EZ04jL19qrrUHJR8A6nK3vJ/GelkDWl4I0VDYSAjLEFQV8g==
Expand Down Expand Up @@ -1042,7 +1047,7 @@
dependencies:
"@textlint/text-to-ast" "^3.3.4"

"@textlint/types@^1.1.2", "@textlint/types@^1.5.4":
"@textlint/types@^1.5.4":
version "1.5.4"
resolved "https://registry.yarnpkg.com/@textlint/types/-/types-1.5.4.tgz#4ebd6b052bb329810949516412ad2b285b04dc02"
integrity sha512-bhSrOjW8AFSa/xf6lYZ2akE0j+4O/WEAA2S/R8RrjNMkA5Az2j57mxPNpqMhEeyHDkpzN/coIlqUwgYvcJHv1A==
Expand All @@ -1059,11 +1064,6 @@
resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==

"@types/structured-source@^3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@types/structured-source/-/structured-source-3.0.0.tgz#e95d76037d400c1957f3b709f023b308e92f3829"
integrity sha512-8u+Wo5+GEXe4jZyQ8TplLp+1A7g32ZcVoE7VZu8VcxnlaEm5I/+T579R7q3qKN76jmK0lRshpo4hl4bj/kEPKA==

"@types/unist@^2.0.0", "@types/unist@^2.0.2":
version "2.0.3"
resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e"
Expand Down Expand Up @@ -1301,6 +1301,11 @@ boundary@^1.0.1:
resolved "https://registry.yarnpkg.com/boundary/-/boundary-1.0.1.tgz#4d67dc2602c0cc16dd9bce7ebf87e948290f5812"
integrity sha1-TWfcJgLAzBbdm85+v4fpSCkPWBI=

boundary@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/boundary/-/boundary-2.0.0.tgz#169c8b1f0d44cf2c25938967a328f37e0a4e5efc"
integrity sha512-rJKn5ooC9u8q13IMCrW0RSp31pxBCHE3y9V/tp3TdWSLf8Em3p6Di4NBpfzbJge9YjjFEsD0RtFEjtvHL5VyEA==

brace-expansion@^1.1.7:
version "1.1.11"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
Expand Down Expand Up @@ -4138,6 +4143,13 @@ structured-source@^3.0.2:
dependencies:
boundary "^1.0.1"

structured-source@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/structured-source/-/structured-source-4.0.0.tgz#0c9e59ee43dedd8fc60a63731f60e358102a4948"
integrity sha512-qGzRFNJDjFieQkl/sVOI2dUjHKRyL9dAJi2gCPGJLbJHBIkyOHxjuocpIEfbLioX+qSJpvbYdT49/YCdMznKxA==
dependencies:
boundary "^2.0.0"

supports-color@6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.0.0.tgz#76cfe742cf1f41bb9b1c29ad03068c05b4c0e40a"
Expand Down Expand Up @@ -4181,15 +4193,14 @@ text-table@^0.2.0:
resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=

textlint-rule-helper@^2.0.0:
version "2.1.1"
resolved "https://registry.yarnpkg.com/textlint-rule-helper/-/textlint-rule-helper-2.1.1.tgz#d572588685359134bc779939b217e61f087dab0f"
integrity sha512-6fxgHzoJVkjl3LaC1b2Egi+5wbhG4i0pU0knJmQujVhxIJ3D3AcQQZPs457xKAi5xKz1WayYeTeJ5jrD/hnO7g==
textlint-rule-helper@^2.3.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/textlint-rule-helper/-/textlint-rule-helper-2.3.0.tgz#5ab84db686d42fd2e39a63b40310501bb336520d"
integrity sha512-Ug78Saahb/qVImttL0NSFyT5/JJ5wXvOPepR2pYAjNi54BsQAAz/hAyyEgKuYeR0+yjFb0KPhby4f880X5vqHA==
dependencies:
"@textlint/ast-node-types" "^4.2.1"
"@textlint/types" "^1.1.2"
structured-source "^3.0.2"
unist-util-visit "^1.1.0"
"@textlint/ast-node-types" "^13.0.2"
structured-source "^4.0.0"
unist-util-visit "^2.0.3"

textlint-scripts@^3.0.0:
version "3.0.0"
Expand Down Expand Up @@ -4217,15 +4228,14 @@ textlint-tester@^5.1.10:
"@textlint/kernel" "^3.4.4"
textlint "^11.9.0"

textlint-util-to-string@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/textlint-util-to-string/-/textlint-util-to-string-3.1.1.tgz#666c8b0f2e00a92b29c8b168b453a9b8aeb48381"
integrity sha512-mHE7/pDw/Hk+Q6YdSMNRrZPl5bCuWnFLbF+bxW+MsWQ64dw+Ia9irkammYbH5I0hVMMcfwb0MQc5nbsjqgWeyQ==
textlint-util-to-string@^3.3.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/textlint-util-to-string/-/textlint-util-to-string-3.3.0.tgz#712fac187edff54e0650ef1a7469db8d5b7c9186"
integrity sha512-HGPpecnYkVkNB5k1NxhzA8nV1T9/ksUj6XK/81U2kb5d1PhPYiGFyikNPI9SxAF7AkrhlQ9mrBgJqy1Q0y4bhA==
dependencies:
"@textlint/ast-node-types" "^4.2.4"
"@types/structured-source" "^3.0.0"
"@textlint/ast-node-types" "^13.0.5"
rehype-parse "^6.0.1"
structured-source "^3.0.2"
structured-source "^4.0.0"
unified "^8.4.0"

textlint@^11.9.0:
Expand Down Expand Up @@ -4441,6 +4451,11 @@ unist-util-is@^3.0.0:
resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-3.0.0.tgz#d9e84381c2468e82629e4a5be9d7d05a2dd324cd"
integrity sha512-sVZZX3+kspVNmLWBPAB6r+7D9ZgAFPNWm66f7YNb420RlQSbn+n8rG8dGZSkrER7ZIXGQYNm5pqC3v3HopH24A==

unist-util-is@^4.0.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-4.1.0.tgz#976e5f462a7a5de73d94b706bac1b90671b57797"
integrity sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==

unist-util-remove-position@^1.0.0:
version "1.1.4"
resolved "https://registry.yarnpkg.com/unist-util-remove-position/-/unist-util-remove-position-1.1.4.tgz#ec037348b6102c897703eee6d0294ca4755a2020"
Expand All @@ -4467,13 +4482,30 @@ unist-util-visit-parents@^2.0.0:
dependencies:
unist-util-is "^3.0.0"

unist-util-visit-parents@^3.0.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz#65a6ce698f78a6b0f56aa0e88f13801886cdaef6"
integrity sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==
dependencies:
"@types/unist" "^2.0.0"
unist-util-is "^4.0.0"

unist-util-visit@^1.1.0:
version "1.4.1"
resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-1.4.1.tgz#4724aaa8486e6ee6e26d7ff3c8685960d560b1e3"
integrity sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==
dependencies:
unist-util-visit-parents "^2.0.0"

unist-util-visit@^2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-2.0.3.tgz#c3703893146df47203bb8a9795af47d7b971208c"
integrity sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==
dependencies:
"@types/unist" "^2.0.0"
unist-util-is "^4.0.0"
unist-util-visit-parents "^3.0.0"

unset-value@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559"
Expand Down

0 comments on commit e3de7e0

Please sign in to comment.