Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ja: add LZ4/Zstd compression support for vector data
zlib isn't implemented yet.
- Loading branch information
Showing
9 changed files
with
751 additions
and
38 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
67 changes: 67 additions & 0 deletions
67
test/command/suite/select/output/lz4/vector/compressed.expected
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| table_create Entries TABLE_PAT_KEY ShortText | ||
| [[0,0.0,0.0],true] | ||
| column_create Entries contents COLUMN_VECTOR|COMPRESS_LZ4 Text | ||
| [[0,0.0,0.0],true] | ||
| load --table Entries | ||
| [ | ||
| { | ||
| "_key": "Groonga", | ||
| "contents": [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." | ||
| ] | ||
| }, | ||
| { | ||
| "_key": "Mroonga", | ||
| "contents": [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." | ||
| ] | ||
| } | ||
| ] | ||
| [[0,0.0,0.0],2] | ||
| select Entries | ||
| [ | ||
| [ | ||
| 0, | ||
| 0.0, | ||
| 0.0 | ||
| ], | ||
| [ | ||
| [ | ||
| [ | ||
| 2 | ||
| ], | ||
| [ | ||
| [ | ||
| "_id", | ||
| "UInt32" | ||
| ], | ||
| [ | ||
| "_key", | ||
| "ShortText" | ||
| ], | ||
| [ | ||
| "contents", | ||
| "Text" | ||
| ] | ||
| ], | ||
| [ | ||
| 1, | ||
| "Groonga", | ||
| [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." | ||
| ] | ||
| ], | ||
| [ | ||
| 2, | ||
| "Mroonga", | ||
| [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." | ||
| ] | ||
| ] | ||
| ] | ||
| ] | ||
| ] |
22 changes: 22 additions & 0 deletions
22
test/command/suite/select/output/lz4/vector/compressed.test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| table_create Entries TABLE_PAT_KEY ShortText | ||
| column_create Entries contents COLUMN_VECTOR|COMPRESS_LZ4 Text | ||
|
|
||
| load --table Entries | ||
| [ | ||
| { | ||
| "_key": "Groonga", | ||
| "contents": [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." | ||
| ] | ||
| }, | ||
| { | ||
| "_key": "Mroonga", | ||
| "contents": [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." | ||
| ] | ||
| } | ||
| ] | ||
|
|
||
| select Entries |
67 changes: 67 additions & 0 deletions
67
test/command/suite/select/output/lz4/vector/packed.expected
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| table_create Entries TABLE_PAT_KEY ShortText | ||
| [[0,0.0,0.0],true] | ||
| column_create Entries contents COLUMN_VECTOR|COMPRESS_LZ4 Text | ||
| [[0,0.0,0.0],true] | ||
| load --table Entries | ||
| [ | ||
| { | ||
| "_key": "Groonga", | ||
| "contents": [ | ||
| "I found Groonga.", | ||
| "It's a fast fulltext search engine!" | ||
| ] | ||
| }, | ||
| { | ||
| "_key": "Mroonga", | ||
| "contents": [ | ||
| "I found Mroonga.", | ||
| "It's a MySQL storage engine to use Groonga!" | ||
| ] | ||
| } | ||
| ] | ||
| [[0,0.0,0.0],2] | ||
| select Entries | ||
| [ | ||
| [ | ||
| 0, | ||
| 0.0, | ||
| 0.0 | ||
| ], | ||
| [ | ||
| [ | ||
| [ | ||
| 2 | ||
| ], | ||
| [ | ||
| [ | ||
| "_id", | ||
| "UInt32" | ||
| ], | ||
| [ | ||
| "_key", | ||
| "ShortText" | ||
| ], | ||
| [ | ||
| "contents", | ||
| "Text" | ||
| ] | ||
| ], | ||
| [ | ||
| 1, | ||
| "Groonga", | ||
| [ | ||
| "I found Groonga.", | ||
| "It's a fast fulltext search engine!" | ||
| ] | ||
| ], | ||
| [ | ||
| 2, | ||
| "Mroonga", | ||
| [ | ||
| "I found Mroonga.", | ||
| "It's a MySQL storage engine to use Groonga!" | ||
| ] | ||
| ] | ||
| ] | ||
| ] | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| table_create Entries TABLE_PAT_KEY ShortText | ||
| column_create Entries contents COLUMN_VECTOR|COMPRESS_LZ4 Text | ||
|
|
||
| load --table Entries | ||
| [ | ||
| { | ||
| "_key": "Groonga", | ||
| "contents": [ | ||
| "I found Groonga.", | ||
| "It's a fast fulltext search engine!" | ||
| ] | ||
| }, | ||
| { | ||
| "_key": "Mroonga", | ||
| "contents": [ | ||
| "I found Mroonga.", | ||
| "It's a MySQL storage engine to use Groonga!" | ||
| ] | ||
| } | ||
| ] | ||
|
|
||
| select Entries |
67 changes: 67 additions & 0 deletions
67
test/command/suite/select/output/zstd/vector/compressed.expected
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| table_create Entries TABLE_PAT_KEY ShortText | ||
| [[0,0.0,0.0],true] | ||
| column_create Entries contents COLUMN_VECTOR|COMPRESS_ZSTD Text | ||
| [[0,0.0,0.0],true] | ||
| load --table Entries | ||
| [ | ||
| { | ||
| "_key": "Groonga", | ||
| "contents": [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." | ||
| ] | ||
| }, | ||
| { | ||
| "_key": "Mroonga", | ||
| "contents": [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." | ||
| ] | ||
| } | ||
| ] | ||
| [[0,0.0,0.0],2] | ||
| select Entries | ||
| [ | ||
| [ | ||
| 0, | ||
| 0.0, | ||
| 0.0 | ||
| ], | ||
| [ | ||
| [ | ||
| [ | ||
| 2 | ||
| ], | ||
| [ | ||
| [ | ||
| "_id", | ||
| "UInt32" | ||
| ], | ||
| [ | ||
| "_key", | ||
| "ShortText" | ||
| ], | ||
| [ | ||
| "contents", | ||
| "Text" | ||
| ] | ||
| ], | ||
| [ | ||
| 1, | ||
| "Groonga", | ||
| [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." | ||
| ] | ||
| ], | ||
| [ | ||
| 2, | ||
| "Mroonga", | ||
| [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." | ||
| ] | ||
| ] | ||
| ] | ||
| ] | ||
| ] |
22 changes: 22 additions & 0 deletions
22
test/command/suite/select/output/zstd/vector/compressed.test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| table_create Entries TABLE_PAT_KEY ShortText | ||
| column_create Entries contents COLUMN_VECTOR|COMPRESS_ZSTD Text | ||
|
|
||
| load --table Entries | ||
| [ | ||
| { | ||
| "_key": "Groonga", | ||
| "contents": [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Groonga is a fast and accurate full text search engine based on inverted index. One of the characteristics of Groonga is that a newly registered document instantly appears in search results. Also, Groonga allows updates without read locks. These characteristics result in superior performance on real-time applications." | ||
| ] | ||
| }, | ||
| { | ||
| "_key": "Mroonga", | ||
| "contents": [ | ||
| "(256 bytes or more required for compression.)", | ||
| "Mroonga is a storage engine for MySQL. It provides fast fulltext search feature for all languages including Chinese, Japanese and Korean to all MySQL users. Mroonga was called Groonga storage engine. Mroonga is Tritonn successor." | ||
| ] | ||
| } | ||
| ] | ||
|
|
||
| select Entries |
67 changes: 67 additions & 0 deletions
67
test/command/suite/select/output/zstd/vector/packed.expected
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| table_create Entries TABLE_PAT_KEY ShortText | ||
| [[0,0.0,0.0],true] | ||
| column_create Entries contents COLUMN_VECTOR|COMPRESS_ZSTD Text | ||
| [[0,0.0,0.0],true] | ||
| load --table Entries | ||
| [ | ||
| { | ||
| "_key": "Groonga", | ||
| "contents": [ | ||
| "I found Groonga.", | ||
| "It's a fast fulltext search engine!" | ||
| ] | ||
| }, | ||
| { | ||
| "_key": "Mroonga", | ||
| "contents": [ | ||
| "I found Mroonga.", | ||
| "It's a MySQL storage engine to use Groonga!" | ||
| ] | ||
| } | ||
| ] | ||
| [[0,0.0,0.0],2] | ||
| select Entries | ||
| [ | ||
| [ | ||
| 0, | ||
| 0.0, | ||
| 0.0 | ||
| ], | ||
| [ | ||
| [ | ||
| [ | ||
| 2 | ||
| ], | ||
| [ | ||
| [ | ||
| "_id", | ||
| "UInt32" | ||
| ], | ||
| [ | ||
| "_key", | ||
| "ShortText" | ||
| ], | ||
| [ | ||
| "contents", | ||
| "Text" | ||
| ] | ||
| ], | ||
| [ | ||
| 1, | ||
| "Groonga", | ||
| [ | ||
| "I found Groonga.", | ||
| "It's a fast fulltext search engine!" | ||
| ] | ||
| ], | ||
| [ | ||
| 2, | ||
| "Mroonga", | ||
| [ | ||
| "I found Mroonga.", | ||
| "It's a MySQL storage engine to use Groonga!" | ||
| ] | ||
| ] | ||
| ] | ||
| ] | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| table_create Entries TABLE_PAT_KEY ShortText | ||
| column_create Entries contents COLUMN_VECTOR|COMPRESS_ZSTD Text | ||
|
|
||
| load --table Entries | ||
| [ | ||
| { | ||
| "_key": "Groonga", | ||
| "contents": [ | ||
| "I found Groonga.", | ||
| "It's a fast fulltext search engine!" | ||
| ] | ||
| }, | ||
| { | ||
| "_key": "Mroonga", | ||
| "contents": [ | ||
| "I found Mroonga.", | ||
| "It's a MySQL storage engine to use Groonga!" | ||
| ] | ||
| } | ||
| ] | ||
|
|
||
| select Entries |