Skip to content

Commit

Permalink
Prefix files support (#124)
Browse files Browse the repository at this point in the history 
* * Reverting change to send_modifieds to lock the map mutex around the transport mutex

* * Added support for using files of required prefixes for loading checkpoints, saving checkpoints, and printing
  * stk_inspect now has lcpl and kpl options for shaping loads or prints
  * karl now has lcpl, kpl, and scpl for shaping loads, prints, and saves
  • Loading branch information
@jredmondson
jredmondson committed Oct 13, 2018
1 parent 88cde94 commit 70cb7cc
Show file tree
Hide file tree
Showing 2 changed files with 126 additions and 4 deletions.
78 changes: 75 additions & 3 deletions tools/karl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include "madara/threads/Threader.h"

#include "madara/utility/Utility.h"
#include "madara/utility/NamedVectorCombinator.h"
#include "madara/filters/GenericFilters.h"
#include "madara/filters/PrefixPrint.h"
#include "madara/logger/GlobalLogger.h"
Expand Down Expand Up @@ -382,13 +383,23 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)
" [-k|--print-knowledge] print final knowledge\n"
" [-kp|--print-prefix pfx] filter prints by prefix. Can be "
"multiple.\n"
" [-kpl|--print-prefix-list file] filter prints by prefix. The "
"file is a \n"
" new-line delimited prefix list that "
"can be read\n"
" by NamedVectorCombinator.\n"
" [-ky] print knowledge after frequent "
"evaluations\n"
" [-l|--level level] the logger level (0+, higher is higher "
"detail)\n"
" [-lcp|--load-checkpoint-prefix prfx]\n"
" prefix of knowledge to load from "
"checkpoint\n"
" [-lcpl|--load-prefix-list file] filter loads by prefix. The "
"file is a \n"
" new-line delimited prefix list that "
"can be read\n"
" by NamedVectorCombinator.\n"
" [-ls|--load-size bytes] size of buffer needed for file load\n"
" [-lt|--load-transport file] a file to load transport settings "
"from\n"
Expand Down Expand Up @@ -425,6 +436,11 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)
" [-scp|--save-checkpoint-prefix prfx]\n"
" prefix of knowledge to save in "
"checkpoint\n"
" [-scpl|--save-prefix-list file] filter saves by prefix. The "
"file is a \n"
" new-line delimited prefix list that "
"can be read\n"
" by NamedVectorCombinator.\n"
" [-sj|--save-json file] save the resulting knowledge base as "
"JSON\n"
" [-sff|--stream-from file] stream knowledge from a file\n"
Expand Down Expand Up @@ -500,15 +516,41 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)
{
print_knowledge = true;
}
else if(arg1 == "-kp" || arg1 == "--print-prefixes")
else if(arg1 == "-kp" || arg1 == "--print-prefix")
{
if(i + 1 < argc)
{
for(int j = i + 1;
j < argc && strlen(argv[j]) > 0 && argv[j][0] != '-'; ++i, ++j)
{
if(debug)
{
std::cout << " Limiting results to prefix " << argv[j] << "\n";
}

print_prefixes.push_back(argv[j]);
}

// ensure uniqueness
utility::NamedVectorCombinator combinator;
combinator.add("current", print_prefixes);
combinator.merge({"current"}, print_prefixes);
}
}
else if(arg1 == "-kpl" || arg1 == "--print-prefix-list")
{
if(i + 1 < argc)
{
// add file and ensure uniqueness
utility::NamedVectorCombinator combinator;
combinator.add("current", print_prefixes);
combinator.from_file("new", argv[i + 1]);

combinator.merge({"current", "new"}, print_prefixes);
}

++i;
}
else if(arg1 == "-ky")
{
print_knowledge_frequency = true;
Expand All @@ -534,6 +576,21 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)

++i;
}
else if(arg1 == "-lcpl" || arg1 == "--load-prefix-list")
{
if(i + 1 < argc)
{
// add file and ensure uniqueness
utility::NamedVectorCombinator combinator;
combinator.add("current", print_prefixes);
combinator.from_file("new", argv[i + 1]);

combinator.merge({"current", "new"},
load_checkpoint_settings.prefixes);
}

++i;
}
else if(arg1 == "-ls" || arg1 == "--load-size")
{
if(i + 1 < argc)
Expand Down Expand Up @@ -843,6 +900,21 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)

++i;
}
else if(arg1 == "-scpl" || arg1 == "--save-prefix-list")
{
if(i + 1 < argc)
{
// add file and ensure uniqueness
utility::NamedVectorCombinator combinator;
combinator.add("current", print_prefixes);
combinator.from_file("new", argv[i + 1]);

combinator.merge({"current", "new"},
save_checkpoint_settings.prefixes);
}

++i;
}
else if(arg1 == "-sff" || arg1 == "--stream-from")
{
if(i + 1 < argc)
Expand Down Expand Up @@ -1134,7 +1206,7 @@ bool load_config_file(std::string full_path, size_t recursion_limit)

// read each line of the text formatted file in, each line contains
// one flag followed by a space then the param if there is a parameter
while (std::getline(file, flag_param))
while(std::getline(file, flag_param))
{
flag = flag_param.substr(0, flag_param.find_first_of(" "));
if(flag_param.find_first_of(" ") == std::string::npos)
Expand Down Expand Up @@ -1247,7 +1319,7 @@ int main(int argc, char** argv)
madara_logger_ptr_log(logger::global_logger.get(), logger::LOG_TRACE,
"Attempting to load default karl config...\n");

if(utility::file_exists (default_karl_config))
if(utility::file_exists(default_karl_config))
{
load_config_file(default_karl_config);
}
Expand Down
52 changes: 51 additions & 1 deletion tools/stk_inspect.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "madara/threads/Threader.h"

#include "madara/utility/Utility.h"
#include "madara/utility/NamedVectorCombinator.h"
#include "madara/logger/GlobalLogger.h"
#include "madara/knowledge/containers/FlexMap.h"

Expand Down Expand Up @@ -212,6 +213,11 @@ int64_t process_command(const std::string & command,
print_prefixes.push_back(tokens[i]);
++result;
}

// ensure uniqueness
utility::NamedVectorCombinator combinator;
combinator.add("current", print_prefixes);
combinator.merge({"current"}, print_prefixes);
}
else if(utility::begins_with(logic, "clear_prefixes"))
{
Expand Down Expand Up @@ -637,7 +643,7 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)
{
print_knowledge = true;
}
else if(arg1 == "-kp" || arg1 == "--print-prefixes")
else if(arg1 == "-kp" || arg1 == "--print-prefix")
{
if(i + 1 < argc)
{
Expand All @@ -651,8 +657,27 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)

print_prefixes.push_back(argv[j]);
}

// ensure uniqueness
utility::NamedVectorCombinator combinator;
combinator.add("current", print_prefixes);
combinator.merge({"current"}, print_prefixes);
}
}
else if(arg1 == "-kpl" || arg1 == "--print-prefix-list")
{
if(i + 1 < argc)
{
// add file and ensure uniqueness
utility::NamedVectorCombinator combinator;
combinator.add("current", print_prefixes);
combinator.from_file("new", argv[i + 1]);

combinator.merge({"current", "new"}, print_prefixes);
}

++i;
}
else if(arg1 == "-ks" || arg1 == "--print-stats")
{
print_stats = true;
Expand Down Expand Up @@ -688,6 +713,21 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)

++i;
}
else if(arg1 == "-lcpl" || arg1 == "--load-prefix-list")
{
if(i + 1 < argc)
{
// add file and ensure uniqueness
utility::NamedVectorCombinator combinator;
combinator.add("current", print_prefixes);
combinator.from_file("new", argv[i + 1]);

combinator.merge({"current", "new"},
load_checkpoint_settings.prefixes);
}

++i;
}
else if(arg1 == "-ls" || arg1 == "--load-size")
{
if(i + 1 < argc)
Expand Down Expand Up @@ -915,12 +955,22 @@ void handle_arguments(int argc, const char** argv, size_t recursion_limit = 10)
" [-k|--print-knowledge] print final knowledge\n"
" [-kp|--print-prefix pfx] filter prints by prefix. Can be "
"multiple.\n"
" [-kpl|--print-prefix-list file] filter prints by prefix. The "
"file is a \n"
" new-line delimited prefix list that "
"can be read\n"
" by NamedVectorCombinator.\n"
" [-ks|--print-stats] print stats knowledge base contents\n"
" [-l|--level level] the logger level(0+, higher is higher "
"detail)\n"
" [-lcp|--load-checkpoint-prefix prfx]\n"
" prefix of knowledge to load from "
"checkpoint\n"
" [-lcpl|--load-prefix-list file] filter loads by prefix. The "
"file is a \n"
" new-line delimited prefix list that "
"can be read\n"
" by NamedVectorCombinator.\n"
" [-ls|--load-size bytes] size of buffer needed for file load\n"
" [-n|--capnp tag:msg_type] register tag with given message "
"schema.\n"
Expand Down

0 comments on commit 70cb7cc

Please sign in to comment.