Skip to content

Commit

Permalink
Merge pull request #31128 from vespa-engine/havardpe/source-selector-…
Browse files Browse the repository at this point in the history 
…iterator-benchmark

baseline source blender integration in benchmark
  • Loading branch information
@geirst
geirst committed May 6, 2024
2 parents 75c398d + 037a40a commit fdc01d3
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 24 deletions.
42 changes: 29 additions & 13 deletions searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@

#include "intermediate_blueprint_factory.h"
#include <vespa/searchlib/queryeval/intermediate_blueprints.h>
#include <vespa/searchlib/attribute/singlenumericattribute.h>
#include <iomanip>
#include <sstream>

namespace search::queryeval::test {

template <typename BlueprintType>
char
IntermediateBlueprintFactory<BlueprintType>::child_name(void* blueprint) const
IntermediateBlueprintFactory::child_name(void* blueprint) const
{
auto itr = _child_names.find(blueprint);
if (itr != _child_names.end()) {
Expand All @@ -18,35 +18,33 @@ IntermediateBlueprintFactory<BlueprintType>::child_name(void* blueprint) const
return '?';
}

template <typename BlueprintType>
IntermediateBlueprintFactory<BlueprintType>::IntermediateBlueprintFactory(vespalib::stringref name)
IntermediateBlueprintFactory::IntermediateBlueprintFactory(vespalib::stringref name)
: _name(name),
_children(),
_child_names()
{
}

template <typename BlueprintType>
IntermediateBlueprintFactory<BlueprintType>::~IntermediateBlueprintFactory() = default;
IntermediateBlueprintFactory::~IntermediateBlueprintFactory() = default;

template <typename BlueprintType>
std::unique_ptr<Blueprint>
IntermediateBlueprintFactory<BlueprintType>::make_blueprint()
IntermediateBlueprintFactory::make_blueprint()
{
auto res = std::make_unique<BlueprintType>();
auto res = make_self();
_child_names.clear();
char name = 'A';
uint32_t source = 1;
for (const auto& factory : _children) {
auto child = factory->make_blueprint();
_child_names[child.get()] = name++;
child->setSourceId(source++); // ignored by non-source-blender blueprints
res->addChild(std::move(child));
}
return res;
}

template <typename BlueprintType>
vespalib::string
IntermediateBlueprintFactory<BlueprintType>::get_name(Blueprint& blueprint) const
IntermediateBlueprintFactory::get_name(Blueprint& blueprint) const
{
auto* intermediate = blueprint.asIntermediate();
if (intermediate != nullptr) {
Expand All @@ -69,11 +67,29 @@ IntermediateBlueprintFactory<BlueprintType>::get_name(Blueprint& blueprint) cons
return get_class_name(blueprint);
}

template class IntermediateBlueprintFactory<AndBlueprint>;
//-----------------------------------------------------------------------------

AndBlueprintFactory::AndBlueprintFactory()
: IntermediateBlueprintFactory<AndBlueprint>("AND")
: IntermediateBlueprintFactory("AND")
{}

std::unique_ptr<IntermediateBlueprint>
AndBlueprintFactory::make_self() const
{
return std::make_unique<AndBlueprint>();
}

//-----------------------------------------------------------------------------

SourceBlenderBlueprintFactory::SourceBlenderBlueprintFactory()
: IntermediateBlueprintFactory("SB"),
_selector(250, "my_source_blender", 1000)
{}

std::unique_ptr<IntermediateBlueprint>
SourceBlenderBlueprintFactory::make_self() const
{
return std::make_unique<SourceBlenderBlueprint>(_selector);
}

}
25 changes: 21 additions & 4 deletions searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,23 @@

#include "benchmark_blueprint_factory.h"
#include <vespa/searchlib/queryeval/intermediate_blueprints.h>
#include <vespa/searchlib/attribute/fixedsourceselector.h>
#include <unordered_map>

namespace search::queryeval::test {

/**
* Factory that creates an IntermediateBlueprint (of the given type) with children created by the given factories.
*/
template <typename BlueprintType>
class IntermediateBlueprintFactory : public BenchmarkBlueprintFactory {
private:
vespalib::string _name;
std::vector<std::shared_ptr<BenchmarkBlueprintFactory>> _children;
std::unordered_map<void*, char> _child_names;

char child_name(void* blueprint) const;

protected:
virtual std::unique_ptr<IntermediateBlueprint> make_self() const = 0;
public:
IntermediateBlueprintFactory(vespalib::stringref name);
~IntermediateBlueprintFactory();
Expand All @@ -30,10 +31,26 @@ class IntermediateBlueprintFactory : public BenchmarkBlueprintFactory {
vespalib::string get_name(Blueprint& blueprint) const override;
};

class AndBlueprintFactory : public IntermediateBlueprintFactory<AndBlueprint> {
class AndBlueprintFactory : public IntermediateBlueprintFactory {
protected:
std::unique_ptr<IntermediateBlueprint> make_self() const override;
public:
AndBlueprintFactory();
};

}
class SourceBlenderBlueprintFactory : public IntermediateBlueprintFactory
{
private:
FixedSourceSelector _selector;
protected:
std::unique_ptr<IntermediateBlueprint> make_self() const override;
public:
SourceBlenderBlueprintFactory();
void init_selector(auto f, uint32_t limit) {
for (uint32_t i = 0; i < limit; ++i) {
_selector.setSource(i, f(i));
}
}
};

}
34 changes: 27 additions & 7 deletions searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -748,23 +748,22 @@ struct BlueprintFactorySetup {

BlueprintFactorySetup::~BlueprintFactorySetup() = default;

template <typename IntermediateBlueprintFactoryType>
void
run_intermediate_blueprint_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs)
run_intermediate_blueprint_benchmark(auto factory_factory, const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs)
{
print_intermediate_blueprint_result_header(2);
double max_speedup = 0.0;
double min_speedup = std::numeric_limits<double>::max();
for (double b_hit_ratio: b.op_hit_ratios) {
auto b_factory = b.make_factory_shared(num_docs, b_hit_ratio);
for (double a_hit_ratio : a.op_hit_ratios) {
IntermediateBlueprintFactoryType factory;
factory.add_child(a.make_factory(num_docs, a_hit_ratio));
factory.add_child(b_factory);
auto factory = factory_factory();
factory->add_child(a.make_factory(num_docs, a_hit_ratio));
factory->add_child(b_factory);
double time_ms_esti = 0.0;
for (auto algo: {PlanningAlgo::Order, PlanningAlgo::Estimate, PlanningAlgo::Cost,
PlanningAlgo::CostForceStrict}) {
auto res = benchmark_search(factory, num_docs + 1, true, false, false, 1.0, algo);
auto res = benchmark_search(*factory, num_docs + 1, true, false, false, 1.0, algo);
print_intermediate_blueprint_result(res, {a_hit_ratio, b_hit_ratio}, algo, num_docs);
if (algo == PlanningAlgo::Estimate) {
time_ms_esti = res.time_ms;
Expand All @@ -789,7 +788,19 @@ void
run_and_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs)
{
std::cout << "AND[A={" << a.to_string() << "},B={" << b.to_string() << "}]" << std::endl;
run_intermediate_blueprint_benchmark<AndBlueprintFactory>(a, b, num_docs);
run_intermediate_blueprint_benchmark([](){ return std::make_unique<AndBlueprintFactory>(); }, a, b, num_docs);
}

void
run_source_blender_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs)
{
std::cout << "SB[A={" << a.to_string() << "},B={" << b.to_string() << "}]" << std::endl;
auto factory_factory = [&](){
auto factory = std::make_unique<SourceBlenderBlueprintFactory>();
factory->init_selector([](uint32_t i){ return (i%10 == 0) ? 1 : 2; }, num_docs + 1);
return factory;
};
run_intermediate_blueprint_benchmark(factory_factory, a, b, num_docs);
}

//-------------------------------------------------------------------------------------
Expand Down Expand Up @@ -973,6 +984,15 @@ TEST(IteratorBenchmark, analyze_AND_bitvector_vs_IN)
}
}

TEST(IteratorBenchmark, analyze_strict_SOURCEBLENDER_memory_and_disk)
{
for (double small_ratio: {0.001, 0.005, 0.01, 0.05}) {
run_source_blender_benchmark({str_fs, QueryOperator::Term, {small_ratio}},
{str_index, QueryOperator::Term, {small_ratio * 10}},
num_docs);
}
}

TEST(IteratorBenchmark, analyze_OR_non_strict_fs)
{
for (auto or_hit_ratio : {0.01, 0.1, 0.5}) {
Expand Down

0 comments on commit fdc01d3

Please sign in to comment.