Skip to content

Commit

Permalink
changed default value for charset_table index option to non_cjk as do…
Browse files Browse the repository at this point in the history 
…cumentation said; fixed #2129; fixed model at test 353; fixed tests 18, 168, 177, 183, 192, 207, 214
  • Loading branch information
@tomatolog
tomatolog committed Aug 25, 2021
1 parent 2cf0838 commit a9638f3
Show file tree
Hide file tree
Showing 9 changed files with 13 additions and 7 deletions.
2 changes: 1 addition & 1 deletion src/indexsettings.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ void CSphTokenizerSettings::Setup ( const CSphConfigSection & hIndex, CSphString
sWarning = "ngram_chars specified, but ngram_len=0; IGNORED";
}

m_sCaseFolding = hIndex.GetStr ( "charset_table", SPHINX_DEFAULT_UTF8_TABLE );
m_sCaseFolding = hIndex.GetStr ( "charset_table", "non_cjk" );
m_iMinWordLen = Max ( hIndex.GetInt ( "min_word_len", 1 ), 1 );
m_sNgramChars = hIndex.GetStr ( "ngram_chars" );
m_sSynonymsFile = hIndex.GetStr ( "exceptions" ); // new option name
Expand Down
1 change: 1 addition & 0 deletions test/test_018/test.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ index test_idx

ngram_len = 1
ngram_chars = U+3000..U+2FA1F
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}
</Config>

Expand Down
3 changes: 2 additions & 1 deletion test/test_168/test.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,13 @@ index idx_blend1
morphology = stem_en,stem_ru
blend_chars = .
blend_mode = trim_both
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}

index idx_blend2
{
type = template
min_word_len = 2
charset_type = utf-8
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+0401->U+0435, U+0451->U+0435, U+A8->U+E5, U+B8->U+E5
ignore_chars = U+AD
min_prefix_len = 2
Expand Down Expand Up @@ -99,6 +99,7 @@ index idx32
docinfo = extern
path = <data_path/>/idx32
dict = keywords
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}

</config>
Expand Down
1 change: 1 addition & 0 deletions test/test_177/test.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ index test
source = test
path = <data_path/>/test
dict = keywords
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}
</config>

Expand Down
2 changes: 2 additions & 0 deletions test/test_183/test.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ index test
{
source = srctest
path = <data_path/>/test
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}
source srctests
{
Expand All @@ -39,6 +40,7 @@ index tests
{
source = srctests
path = <data_path/>/tests
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}

index rt28
Expand Down
1 change: 1 addition & 0 deletions test/test_192/test.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ index wb

wordforms = <this_test/>/wordforms-wb.txt
blend_chars = U+2D
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}


Expand Down
2 changes: 2 additions & 0 deletions test/test_207/test.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ index test1
morphology = lemmatize_ru_all
blend_chars = @, .
wordforms = <this_test/>/wordforms.txt
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}

index test2 : test1
Expand Down Expand Up @@ -65,6 +66,7 @@ index testrt1
wordforms = <this_test/>/wordforms.txt
rt_attr_uint = gid
rt_field = title
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}

index testrt2 : testrt1
Expand Down
2 changes: 2 additions & 0 deletions test/test_214/test.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ index rt
<variant>index_field_lengths = 0</variant>
<variant>index_field_lengths = 1</variant>
</dynamic>

charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451
}
</config>

Expand Down
6 changes: 1 addition & 5 deletions test/test_353/model.bin
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1 @@
a:1:{i:0;a:18:{i:0;a:3:{s:8:"sphinxql";s:25:"show index delta settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:104:"charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451";}}}i:1;a:3:{s:8:"sphinxql";s:19:"select * from main1";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:2:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:2;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:4;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:2;a:3:{s:8:"sphinxql";s:19:"select * from main2";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:2:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:2;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:4;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:3;a:3:{s:8:"sphinxql";s:19:"select * from delta";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:2:"id";s:1:"6";s:3:"gid";s:1:"3";}}}i:4;a:2:{s:8:"sphinxql";s:44:"alter table delta killlist_target='main1:kl'";s:14:"total_affected";i:0;}i:5;a:3:{s:8:"sphinxql";s:25:"show index delta settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:131:"killlist_target = main1:kl
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451";}}}i:6;a:3:{s:8:"sphinxql";s:19:"select * from main1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:7;a:3:{s:8:"sphinxql";s:19:"select * from main2";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:2:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:2;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:4;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:8;a:2:{s:8:"sphinxql";s:44:"alter table delta killlist_target='main2:kl'";s:14:"total_affected";i:0;}i:9;a:3:{s:8:"sphinxql";s:25:"show index delta settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:131:"killlist_target = main2:kl
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451";}}}i:10;a:3:{s:8:"sphinxql";s:19:"select * from main1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:11;a:3:{s:8:"sphinxql";s:19:"select * from main2";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:12;a:2:{s:8:"sphinxql";s:65:"alter table delta killlist_target='main1:kl, main2:kl, main3:kl'";s:14:"total_affected";i:0;}i:13;a:3:{s:8:"sphinxql";s:9:"show meta";s:10:"total_rows";i:4;s:4:"rows";a:3:{i:0;a:2:{s:13:"Variable_name";s:7:"warning";s:5:"Value";s:44:"index 'main3' from killlist_target not found";}i:1;a:2:{s:13:"Variable_name";s:5:"total";s:5:"Value";s:1:"0";}i:2;a:2:{s:13:"Variable_name";s:11:"total_found";s:5:"Value";s:1:"0";}}}i:14;a:3:{s:8:"sphinxql";s:25:"show index delta settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:149:"killlist_target = main1:kl,main2:kl,main3:kl
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451";}}}i:15;a:3:{s:8:"sphinxql";s:19:"select * from main1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:16;a:3:{s:8:"sphinxql";s:19:"select * from main2";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:17;a:3:{s:8:"sphinxql";s:26:"show index delta2 settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:131:"killlist_target = main1:kl
charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451";}}}}}
a:1:{i:0;a:18:{i:0;a:3:{s:8:"sphinxql";s:25:"show index delta settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:0:"";}}}i:1;a:3:{s:8:"sphinxql";s:19:"select * from main1";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:2:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:2;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:4;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:2;a:3:{s:8:"sphinxql";s:19:"select * from main2";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:2:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:2;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:4;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:3;a:3:{s:8:"sphinxql";s:19:"select * from delta";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:2:"id";s:1:"6";s:3:"gid";s:1:"3";}}}i:4;a:2:{s:8:"sphinxql";s:44:"alter table delta killlist_target='main1:kl'";s:14:"total_affected";i:0;}i:5;a:3:{s:8:"sphinxql";s:25:"show index delta settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:26:"killlist_target = main1:kl";}}}i:6;a:3:{s:8:"sphinxql";s:19:"select * from main1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:7;a:3:{s:8:"sphinxql";s:19:"select * from main2";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:2:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:2;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:4;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:8;a:2:{s:8:"sphinxql";s:44:"alter table delta killlist_target='main2:kl'";s:14:"total_affected";i:0;}i:9;a:3:{s:8:"sphinxql";s:25:"show index delta settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:26:"killlist_target = main2:kl";}}}i:10;a:3:{s:8:"sphinxql";s:19:"select * from main1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:11;a:3:{s:8:"sphinxql";s:19:"select * from main2";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:12;a:2:{s:8:"sphinxql";s:65:"alter table delta killlist_target='main1:kl, main2:kl, main3:kl'";s:14:"total_affected";i:0;}i:13;a:3:{s:8:"sphinxql";s:9:"show meta";s:10:"total_rows";i:4;s:4:"rows";a:3:{i:0;a:2:{s:13:"Variable_name";s:7:"warning";s:5:"Value";s:44:"index 'main3' from killlist_target not found";}i:1;a:2:{s:13:"Variable_name";s:5:"total";s:5:"Value";s:1:"0";}i:2;a:2:{s:13:"Variable_name";s:11:"total_found";s:5:"Value";s:1:"0";}}}i:14;a:3:{s:8:"sphinxql";s:25:"show index delta settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:44:"killlist_target = main1:kl,main2:kl,main3:kl";}}}i:15;a:3:{s:8:"sphinxql";s:19:"select * from main1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:16;a:3:{s:8:"sphinxql";s:19:"select * from main2";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:2:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";}i:1;a:2:{s:2:"id";s:1:"3";s:3:"gid";s:1:"2";}i:2;a:2:{s:2:"id";s:1:"4";s:3:"gid";s:1:"2";}i:3;a:2:{s:2:"id";s:1:"5";s:3:"gid";s:1:"3";}}}i:17;a:3:{s:8:"sphinxql";s:26:"show index delta2 settings";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:13:"Variable_name";s:8:"settings";s:5:"Value";s:26:"killlist_target = main1:kl";}}}}}

0 comments on commit a9638f3

Please sign in to comment.