Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
added first version of bigtesting framework
  • Loading branch information
lambday authored and karlnapf committed Mar 29, 2016
1 parent 85e677b commit 8aeaf74
Show file tree
Hide file tree
Showing 34 changed files with 2,159 additions and 0 deletions.
42 changes: 42 additions & 0 deletions src/shogun/statistics/experimental/internals/BlockwiseDetails.cpp
@@ -0,0 +1,42 @@
/*
* Restructuring Shogun's statistical hypothesis testing framework.
* Copyright (C) 2016 Soumyajit De
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include <shogun/statistics/experimental/internals/BlockwiseDetails.h>

using namespace shogun;
using namespace internal;

BlockwiseDetails::BlockwiseDetails()
: m_blocksize(0), m_num_blocks_per_burst(1), m_max_num_samples_per_burst(0),
m_next_block_index(0), m_total_num_blocks(0)
{
}

BlockwiseDetails& BlockwiseDetails::with_blocksize(index_t blocksize)
{
m_blocksize = blocksize;
m_max_num_samples_per_burst = m_blocksize * m_num_blocks_per_burst;
return *this;
}

BlockwiseDetails& BlockwiseDetails::with_num_blocks_per_burst(index_t num_blocks_per_burst)
{
m_num_blocks_per_burst = num_blocks_per_burst;
m_max_num_samples_per_burst = m_blocksize * m_num_blocks_per_burst;
return *this;
}
51 changes: 51 additions & 0 deletions src/shogun/statistics/experimental/internals/BlockwiseDetails.h
@@ -0,0 +1,51 @@
/*
* Restructuring Shogun's statistical hypothesis testing framework.
* Copyright (C) 2016 Soumyajit De
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include <shogun/lib/common.h>

#ifndef BLOCK_WISE_DETAILS_H__
#define BLOCK_WISE_DETAILS_H__

namespace shogun
{

namespace internal
{

class BlockwiseDetails
{
friend class DataFetcher;
friend class StreamingDataFetcher;
friend class DataManager;
public:
BlockwiseDetails();
BlockwiseDetails& with_blocksize(index_t blocksize);
BlockwiseDetails& with_num_blocks_per_burst(index_t num_blocks_per_burst);
private:
index_t m_blocksize;
index_t m_num_blocks_per_burst;
index_t m_max_num_samples_per_burst;
// the following will be set by data fetchers
index_t m_next_block_index;
index_t m_total_num_blocks;
};

}

}
#endif // BLOCK_WISE_DETAILS_H__
@@ -0,0 +1,94 @@
/*
* Restructuring Shogun's statistical hypothesis testing framework.
* Copyright (C) 2016 Soumyajit De
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include <shogun/lib/SGMatrix.h>
#include <shogun/statistics/experimental/internals/ComputationManager.h>
#include <shogun/statistics/experimental/internals/mmd/UnbiasedFull.h>

using namespace shogun;
using namespace internal;

ComputationManager::ComputationManager()
{
}

ComputationManager::~ComputationManager()
{
}

void ComputationManager::num_data(index_t n)
{
kernel_matrices.resize(n);
}

SGMatrix<float64_t>& ComputationManager::data(index_t i)
{
return kernel_matrices[i];
}

void ComputationManager::enqueue_job(std::function<float64_t(SGMatrix<float64_t>)> job)
{
jobq.push(job);
}

void ComputationManager::compute()
{
while (!jobq.empty())
{
std::vector<float64_t> results;
if (gpu)
{
// TODO results = operation.compute_using_gpu(kernel_matrices);
}
else
{
results.resize(kernel_matrices.size());
#pragma omp parallel for
for (auto i = 0; i < kernel_matrices.size(); ++i)
{
const auto& operation = jobq.front();
results[i] = operation(kernel_matrices[i]);
}
}
resultq.push(results);
jobq.pop();
}
}

std::vector<float64_t> ComputationManager::next_result()
{
std::vector<float64_t> result;
if (!resultq.empty())
{
result = resultq.front();
resultq.pop();
}
return result;
}

ComputationManager& ComputationManager::use_gpu()
{
gpu = true;
return *this;
}

ComputationManager& ComputationManager::use_cpu()
{
gpu = false;
return *this;
}
63 changes: 63 additions & 0 deletions src/shogun/statistics/experimental/internals/ComputationManager.h
@@ -0,0 +1,63 @@
/*
* Restructuring Shogun's statistical hypothesis testing framework.
* Copyright (C) 2016 Soumyajit De
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#ifndef COMPUTATION_MANAGER_H__
#define COMPUTATION_MANAGER_H__

#include <vector>
#include <queue>
#include <functional>
#include <shogun/lib/common.h>

namespace shogun
{

template <typename T> class SGMatrix;

namespace internal
{

class ComputationManager
{
public:
ComputationManager();
~ComputationManager();

void num_data(index_t n);
SGMatrix<float64_t>& data(index_t i);

void enqueue_job(std::function<float64_t(SGMatrix<float64_t>)> job);

void compute();

std::vector<float64_t> next_result();

ComputationManager& use_cpu();
ComputationManager& use_gpu();
private:
bool gpu;
std::vector<SGMatrix<float64_t>> kernel_matrices;
std::queue<std::function<float64_t(SGMatrix<float64_t>)>> jobq;
std::queue<std::vector<float64_t>> resultq;
};

}

}

#endif // COMPUTATION_MANAGER_H__
100 changes: 100 additions & 0 deletions src/shogun/statistics/experimental/internals/DataFetcher.cpp
@@ -0,0 +1,100 @@
/*
* Restructuring Shogun's statistical hypothesis testing framework.
* Copyright (C) 2016 Soumyajit De
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <shogun/features/Features.h>
#include <shogun/statistics/experimental/internals/DataFetcher.h>


using namespace shogun;
using namespace internal;

DataFetcher::DataFetcher() : m_num_samples(0)
{
}

DataFetcher::DataFetcher(CFeatures* samples)
{
SG_REF(samples);
m_samples = std::shared_ptr<CFeatures>(samples, [](CFeatures* ptr) { SG_UNREF(ptr); });
m_num_samples = m_samples->get_num_vectors();
}

DataFetcher::~DataFetcher()
{
}

const char* DataFetcher::get_name() const
{
return "DataFetcher";
}

void DataFetcher::start()
{
if (m_block_details.m_blocksize == 0)
{
m_block_details.with_blocksize(m_num_samples);
}
m_block_details.m_total_num_blocks = m_num_samples / m_block_details.m_blocksize;
reset();
}

std::shared_ptr<CFeatures> DataFetcher::next()
{
auto num_more_samples = m_num_samples - m_block_details.m_next_block_index * m_block_details.m_blocksize;
if (num_more_samples > 0)
{
auto num_samples_this_burst = m_block_details.m_max_num_samples_per_burst;
if (num_samples_this_burst > num_more_samples)
{
num_samples_this_burst = num_more_samples;
}
if (num_samples_this_burst < m_num_samples)
{
m_samples->remove_subset();
SGVector<index_t> inds(num_samples_this_burst);
std::iota(inds.vector, inds.vector + inds.vlen, m_block_details.m_next_block_index * m_block_details.m_blocksize);
m_samples->add_subset(inds);
}

m_block_details.m_next_block_index += m_block_details.m_num_blocks_per_burst;
return m_samples;
}
return nullptr;
}

void DataFetcher::reset()
{
m_block_details.m_next_block_index = 0;
m_samples->remove_all_subsets();
}

void DataFetcher::end()
{
m_samples->remove_all_subsets();
}

const index_t DataFetcher::get_num_samples() const
{
return m_num_samples;
}

BlockwiseDetails& DataFetcher::fetch_blockwise()
{
return m_block_details;
}

0 comments on commit 8aeaf74

Please sign in to comment.