Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add an option to only build PSO and POS. #542

Merged
merged 4 commits into from
Jan 20, 2022
Merged

Add an option to only build PSO and POS. #542

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
94fa0f9
Add an option to only build PSO and POS.
joka921 Jan 14, 2022
2b1d832
Automatically propagate the information that an index only has two pe…
joka921 Jan 20, 2022
4b2ff9a
Also enable building of patterns when no SPO permutation is buil[t|d|…
joka921 Jan 20, 2022
11e87c0
small tweaks from the review.
joka921 Jan 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 9 additions & 4 deletions src/index/Index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,15 @@ void Index::createFromFile(const string& filename) {
// For the first permutation, perform a unique.
createPermutationPair<IndexMetaDataHmapDispatcher>(&vocabData, _PSO, _POS,
PerformUnique::True);
// After the SPO permutation, create patterns if so desired.
createPermutationPair<IndexMetaDataMmapDispatcher>(
&vocabData, _SPO, _SOP, PerformUnique::False, _usePatterns);
createPermutationPair<IndexMetaDataMmapDispatcher>(&vocabData, _OSP, _OPS);
if (_loadAllPermutations) {
// After the SPO permutation, create patterns if so desired.
createPermutationPair<IndexMetaDataMmapDispatcher>(
&vocabData, _SPO, _SOP, PerformUnique::False, _usePatterns);
createPermutationPair<IndexMetaDataMmapDispatcher>(&vocabData, _OSP, _OPS);
_configurationJson["has-all-permutations"] = true;
} else {
_configurationJson["has-all-permutations"] = false;
}
LOG(INFO) << "Finished writing permutations" << std::endl;

// Dump the configuration again in case the permutations have added some
Expand Down
11 changes: 10 additions & 1 deletion src/index/IndexBuilderMain.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ struct option options[] = {
{"keep-temporary-files", no_argument, NULL, 'k'},
{"settings-file", required_argument, NULL, 's'},
{"no-compressed-vocabulary", no_argument, NULL, 'N'},
{"only-pso-and-pos-permutations", no_argument, NULL, 'o'},
{NULL, 0, NULL, 0}};

string getStxxlConfigFileName(const string& location) {
Expand Down Expand Up @@ -138,6 +139,9 @@ void printUsage(char* execName) {
<< "Do NOT use prefix compression on the vocabulary (default is to "
"compress)."
<< endl;
cerr << " " << std::setw(20) << "o, only-pos-and-pso-permutations"
<< std::setw(1) << " "
<< "Only load PSO and POS permutations" << endl;
cerr.copyfmt(cerrState);
}

Expand All @@ -163,10 +167,11 @@ int main(int argc, char** argv) {
bool usePatterns = true;
bool onlyAddTextIndex = false;
bool keepTemporaryFiles = false;
bool loadAllPermutations = true;
optind = 1;
// Process command line arguments.
while (true) {
int c = getopt_long(argc, argv, "F:f:i:w:d:lT:K:hAks:N", options, nullptr);
int c = getopt_long(argc, argv, "F:f:i:w:d:lT:K:hAks:No", options, nullptr);
if (c == -1) {
break;
}
Expand Down Expand Up @@ -214,6 +219,9 @@ int main(int argc, char** argv) {
case 'N':
useCompression = false;
break;
case 'o':
loadAllPermutations = false;
break;
default:
cerr << endl
<< "! ERROR in processing options (getopt returned '" << c
Expand Down Expand Up @@ -263,6 +271,7 @@ int main(int argc, char** argv) {
index.setKeepTempFiles(keepTemporaryFiles);
index.setSettingsFile(settingsFile);
index.setPrefixCompression(useCompression);
index.setLoadAllPermutations(loadAllPermutations);
if (!onlyAddTextIndex) {
// if onlyAddTextIndex is true, we do not want to construct an index,
// but assume that it already exists (especially we need a valid
Expand Down