Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

improve the imputer program #740

Merged
merged 7 commits into from Sep 26, 2016
Merged

improve the imputer program #740

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
645c58d
add overload that imputes every dimensions
keon Jul 25, 2016
7e05bb9
discard redundant parts and optimize Imputer
keon Jul 26, 2016
2a3a1cb
improve comments for impute program
keon Aug 2, 2016
d130f54
fix typo in imputer
keon Aug 2, 2016
0a6ab52
make missing_policy consistent with other policies
keon Aug 2, 2016
6022b4d
fix typo
keon Sep 8, 2016
2df9caf
warn if the dimension does not contain missing value
keon Sep 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
28 changes: 25 additions & 3 deletions src/mlpack/core/data/imputer.hpp
View file
Open in desktop
Expand Up @@ -44,9 +44,9 @@ class Imputer
}

/**
* Given an input dataset, replace missing values with given imputation
* strategy. This overload does not produce output matrix, but overwrites the
* result into the input matrix.
* Given an input dataset, replace missing values of a dimension with given
* imputation strategy. This function does not produce output matrix, but
* overwrites the result into the input matrix.
*
* @param input Input dataset to apply imputation.
* @oaran missingValue User defined missing value; it can be anything.
Expand All @@ -60,6 +60,28 @@ class Imputer
strategy.Impute(input, mappedValue, dimension, columnMajor);
}

/**
* Given an input dataset, replace missing values of all dimensions with given
* imputation strategy. This function does not produce output matrix, but
* overwrites the result into the input matrix.
*
* @param input Input dataset to apply imputation.
* @oaran missingValue User defined missing value; it can be anything.
*/
void Impute(arma::Mat<T>& input,
const std::string& missingValue)
{
const size_t dimensions = columnMajor ? input.n_rows : input.n_cols;
for (size_t i = 0; i < dimensions; ++i)
{
if (mapper.NumMappings(i) > 0)
{
T mappedValue = static_cast<T>(mapper.UnmapValue(missingValue, i));
strategy.Impute(input, mappedValue, i, columnMajor);
}
}
}

//! Get the strategy
const StrategyType& Strategy() const { return strategy; }

Expand Down
106 changes: 57 additions & 49 deletions src/mlpack/methods/preprocess/preprocess_imputer_main.cpp
View file
Open in desktop
Expand Up @@ -99,76 +99,84 @@ int main(int argc, char** argv)
MissingPolicy policy(missingSet);
using MapperType = DatasetMapper<MissingPolicy>;
DatasetMapper<MissingPolicy> info(policy);
std::vector<size_t> dirtyDimensions;

Load(inputFile, input, info, true, true);

// print how many mapping exist in each dimensions
bool isDirty = false;
for (size_t i = 0; i < input.n_rows; ++i)
{
size_t numMappings = info.NumMappings(i);
Log::Info << numMappings << " mappings in dimension " << i << "."
<< endl;
if (numMappings > 0)
{
dirtyDimensions.push_back(i);
Log::Info << "Replacing " << numMappings << " values in dimension " << i
<< "." << endl;
if (!isDirty)
{
isDirty = true;
}
}
}

// Initialize imputer class
Imputer<double, MapperType, MeanImputation<double>> imputer(info);
if (strategy == "mean")
if (!isDirty)
{
Imputer<double, MapperType, MeanImputation<double>> imputer(info);
}
else if (strategy == "median")
{
Imputer<double, MapperType, MedianImputation<double>> imputer(info);
}
else if (strategy == "listwise_deletion")
{
Imputer<double, MapperType, ListwiseDeletion<double>> imputer(info);
}
else if (strategy == "custom")
{
CustomImputation<double> strat(customValue);
Imputer<double, MapperType, CustomImputation<double>> imputer(info, strat);
Log::Info << "The file does not contain any user-defined missing variables."
<< " The program did not perform any imputation." << endl;
}
else
{
Log::Fatal << "'" << strategy << "' imputation strategy does not exist"
<< endl;
}

Timer::Start("imputation");
if (CLI::HasParam("dimension"))
{
// when --dimension is specified,
// the program will apply the changes to only the given dimension.
Log::Info << "Performing '" << strategy << "' imputation strategy "
<< "to replace '" << missingValue << "' on dimension " << dimension
<< "." << endl;
// Initialize imputer class
Imputer<double, MapperType, MeanImputation<double>> imputer(info);
if (strategy == "mean")
{
Imputer<double, MapperType, MeanImputation<double>> imputer(info);
}
else if (strategy == "median")
{
Imputer<double, MapperType, MedianImputation<double>> imputer(info);
}
else if (strategy == "listwise_deletion")
{
Imputer<double, MapperType, ListwiseDeletion<double>> imputer(info);
}
else if (strategy == "custom")
{
CustomImputation<double> strat(customValue);
Imputer<double, MapperType, CustomImputation<double>> imputer(info, strat);
}
else
{
Log::Fatal << "'" << strategy << "' imputation strategy does not exist"
<< endl;
}

imputer.Impute(input, missingValue, dimension);
}
else
{
// when --dimension is not specified,
// the program will apply the changes to all dimensions.
Log::Info << "Performing '" << strategy << "' imputation strategy "
<< "to replace '" << missingValue << "' on all dimensions." << endl;
Timer::Start("imputation");
if (CLI::HasParam("dimension"))
{
// when --dimension is specified,
// the program will apply the changes to only the given dimension.
Log::Info << "Performing '" << strategy << "' imputation strategy "
<< "to replace '" << missingValue << "' on dimension " << dimension
<< "." << endl;

for (size_t i : dirtyDimensions)
imputer.Impute(input, missingValue, dimension);
}
else
{
imputer.Impute(input, missingValue, i);
// when --dimension is not specified,
// the program will apply the changes to all dimensions.
Log::Info << "Performing '" << strategy << "' imputation strategy "
<< "to replace '" << missingValue << "' on all dimensions." << endl;

imputer.Impute(input, missingValue);
}
}
Timer::Stop("imputation");
Timer::Stop("imputation");

if (!outputFile.empty())
{
Log::Info << "Saving results to '" << outputFile << "'." << endl;
Save(outputFile, input, false);
if (!outputFile.empty())
{
Log::Info << "Saving results to '" << outputFile << "'." << endl;
Save(outputFile, input, false);
}
}
}

33 changes: 29 additions & 4 deletions src/mlpack/tests/imputation_test.cpp
View file
Open in desktop
Expand Up @@ -47,12 +47,12 @@ BOOST_AUTO_TEST_CASE(DatasetMapperImputerTest)
DatasetMapper<MissingPolicy> info(policy);
BOOST_REQUIRE(data::Load("test_file.csv", input, info) == true);

// row and column test
// row and column test.
BOOST_REQUIRE_EQUAL(input.n_rows, 3);
BOOST_REQUIRE_EQUAL(input.n_cols, 3);

// Load check
// MissingPolicy should convert strings to nans
// MissingPolicy should convert strings to nans.
BOOST_REQUIRE(std::isnan(input(0, 0)) == true);
BOOST_REQUIRE_CLOSE(input(0, 1), 5.0, 1e-5);
BOOST_REQUIRE_CLOSE(input(0, 2), 8.0, 1e-5);
Expand All @@ -68,10 +68,10 @@ BOOST_AUTO_TEST_CASE(DatasetMapperImputerTest)
Imputer<double,
DatasetMapper<MissingPolicy>,
CustomImputation<double>> imputer(info, customStrategy);
// convert a or nan to 99 for dimension 0
// convert a or nan to 99 for dimension 0.
imputer.Impute(input, "a", 0);

// Custom imputation result check
// Custom imputation result check.
BOOST_REQUIRE_CLOSE(input(0, 0), 99.0, 1e-5);
BOOST_REQUIRE_CLOSE(input(0, 1), 5.0, 1e-5);
BOOST_REQUIRE_CLOSE(input(0, 2), 8.0, 1e-5);
Expand All @@ -82,6 +82,31 @@ BOOST_AUTO_TEST_CASE(DatasetMapperImputerTest)
BOOST_REQUIRE(std::isnan(input(2, 1)) == true); // remains as NaN
BOOST_REQUIRE_CLOSE(input(2, 2), 10.0, 1e-5);

// Now, try impute() overload with all dimensions.
arma::mat allInput;
MissingPolicy allPolicy({"a"});
DatasetMapper<MissingPolicy> allInfo(allPolicy);
BOOST_REQUIRE(data::Load("test_file.csv", allInput, allInfo) == true);

// convert missing vals to 99.
CustomImputation<double> allCustomStrategy(99);
Imputer<double,
DatasetMapper<MissingPolicy>,
CustomImputation<double>> allImputer(allInfo, allCustomStrategy);
// convert a or nan to 99 for all dimensions.
allImputer.Impute(allInput, "a");

// Custom imputation result check
BOOST_REQUIRE_CLOSE(allInput(0, 0), 99.0, 1e-5);
BOOST_REQUIRE_CLOSE(allInput(0, 1), 5.0, 1e-5);
BOOST_REQUIRE_CLOSE(allInput(0, 2), 8.0, 1e-5);
BOOST_REQUIRE_CLOSE(allInput(1, 0), 2.0, 1e-5);
BOOST_REQUIRE_CLOSE(allInput(1, 1), 6.0, 1e-5);
BOOST_REQUIRE_CLOSE(allInput(1, 2), 9.0, 1e-5);
BOOST_REQUIRE_CLOSE(allInput(2, 0), 3.0, 1e-5);
BOOST_REQUIRE_CLOSE(allInput(2, 1), 99.0, 1e-5);
BOOST_REQUIRE_CLOSE(allInput(2, 2), 10.0, 1e-5);

// Remove the file.
remove("test_file.csv");
}
Expand Down