Skip to content

Commit

Permalink
Add MS-SSIM and CM-SSIM for metrics module.
Browse files Browse the repository at this point in the history
  • Loading branch information
@tatsy
tatsy committed Oct 3, 2019
1 parent ba5ca04 commit adfe8af
Show file tree
Hide file tree
Showing 4 changed files with 296 additions and 52 deletions.
48 changes: 48 additions & 0 deletions sources/metrics/basics.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,54 @@ LIME_METHOD_API double PSNR(cv::InputArray I, cv::InputArray J);
*/
LIME_METHOD_API double SSIM(cv::InputArray I, cv::InputArray J);

/**
* Calculate multi-scale structural similarity (MS-SSIM).
* This method is based on the following paper.
* Wang et al., 2003, "Multi-scale Structural Similarity for Image Quaility Assessment"
* @ingroup metrics
*
* @details
* @b Parameters
* @arg @b I: The first input image.
* @arg @b J: The second input image.
*
* @b Python
* @code{.py}
* score = lime.PSNR(I, J)
* @endcode
*
* @b Parameters
* @arg @b I - @c numpy.ndarray : The first input image.
* @arg @b J - @c numpy.ndarray : The second input image.
* @b Returns
* @arg @b score - @c float : The value of MS-SSIM.
*/
LIME_METHOD_API double MSSSIM(cv::InputArray I, cv::InputArray J);

/**
* Calculate color multi-scale structural similarity (CM-SSIM).
* This method is based on the following paper.
* Hassan and Bhagvati 2012, "Structural Similarity Measure for Color Images"
* @ingroup metrics
*
* @details
* @b Parameters
* @arg @b I: The first input image.
* @arg @b J: The second input image.
*
* @b Python
* @code{.py}
* score = lime.PSNR(I, J)
* @endcode
*
* @b Parameters
* @arg @b I - @c numpy.ndarray : The first input image.
* @arg @b J - @c numpy.ndarray : The second input image.
* @b Returns
* @arg @b score - @c float : The value of CM-SSIM.
*/
LIME_METHOD_API double CMSSIM(cv::InputArray I, cv::InputArray J);

} // namespace lime

#ifndef LIME_USE_LIBRARY
Expand Down
187 changes: 156 additions & 31 deletions sources/metrics/basics_detail.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,51 @@

namespace lime {

namespace {

void calsSSIM_LCS(const cv::Mat& I, const cv::Mat& J, double *l, double *c, double *s) {
const int width = I.cols;
const int height = I.rows;
const int totalSize = width * height;

double mu_x = 0.0, mu_y = 0.0;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
mu_x += (double)I.at<uchar>(y, x);
mu_y += (double)J.at<uchar>(y, x);
}
}
mu_x /= totalSize;
mu_y /= totalSize;

double sig_x = 0.0, sig_y = 0.0, cov_xy = 0.0;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
const double diff_x = (double)I.at<uchar>(x, y) - mu_x;
const double diff_y = (double)J.at<uchar>(x, y) - mu_y;
sig_x += diff_x * diff_x;
sig_y += diff_y * diff_y;
cov_xy += diff_x * diff_y;
}
}
sig_x = std::sqrt(sig_x / totalSize);
sig_y = std::sqrt(sig_y / totalSize);
cov_xy /= totalSize;

const double k1 = 0.01;
const double k2 = 0.03;
const double L = 255.0;
const double c1 = (k1 * L) * (k1 * L);
const double c2 = (k2 * L) * (k2 * L);
const double c3 = 0.5 * c2;

*l = (2.0 * mu_x * mu_y + c1) / (mu_x * mu_x + mu_y * mu_y + c1);
*c = (2.0 * sig_x * sig_y + c2) / (sig_x * sig_x + sig_y * sig_y + c2);
*s = (cov_xy + c3) / (sig_x * sig_y + c3);
}

} // anonymous namespace

double MSE(cv::InputArray I_, cv::InputArray J_) {
cv::Mat I = I_.getMat();
cv::Mat J = J_.getMat();
Expand Down Expand Up @@ -93,46 +138,126 @@ double SSIM(cv::InputArray I_, cv::InputArray J_) {
cv::cvtColor(J, J, cv::COLOR_BGR2GRAY);
}

const int width = I.cols;
const int height = I.rows;
const int totalSize = width * height;
double l, c, s;
calsSSIM_LCS(I, J, &l, &c, &s);

double mu_x = 0.0, mu_y = 0.0;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
mu_x += (double)I.at<uchar>(y, x);
mu_y += (double)J.at<uchar>(y, x);
return l * c * s;
}

double MSSSIM(cv::InputArray I_, cv::InputArray J_) {
cv::Mat I = I_.getMat();
cv::Mat J = J_.getMat();

Assertion(I.cols == J.cols && I.rows == J.rows, "Sizes of two images must be the same!");
Assertion(I.depth() == J.depth(), "Bit depths of two images must be the same!");
Assertion((I.channels() == 1 && J.channels() == 1) ||
(I.channels() == 3 && J.channels() == 3),
"Two images must have 1 or 3 channels!");
Assertion((I.depth() == CV_8U && J.depth() == CV_8U) ||
(I.depth() == CV_32F && J.depth() == CV_32F),
"Two images must be with 8-bit unsigned integer or 32-bit floating point number");

if (I.depth() == CV_32F) {
I.convertTo(I, CV_8U, 255.0);
}

if (J.depth() == CV_32F) {
J.convertTo(J, CV_8U, 255.0);
}

if (I.channels() == 3) {
cv::cvtColor(I, I, cv::COLOR_BGR2GRAY);
}

if (J.channels() == 3) {
cv::cvtColor(J, J, cv::COLOR_BGR2GRAY);
}

const double alpha = 0.1333;
const double beta[] = { 0.0448, 0.2856, 0.3001, 0.2363, 0.1333 };
const double gamma[] = { 0.0448, 0.2856, 0.3001, 0.2363, 0.1333 };
const int M = 5;

cv::Mat subI, subJ;
I.copyTo(subI);
J.copyTo(subJ);

double score = 1.0;
for (int i = 0; i < M; i++) {
double l, c, s;
calsSSIM_LCS(subI, subJ, &l, &c, &s);

score *= std::pow(c, beta[i]) * std::pow(s, gamma[i]);
if (i == M - 1) {
score *= std::pow(l, alpha);
break;
}

cv::pyrDown(subI, subI);
cv::pyrDown(subJ, subJ);
}
mu_x /= totalSize;
mu_y /= totalSize;

double sig_x = 0.0, sig_y = 0.0, cov_xy = 0.0;
return score;
}

double CMSSIM(cv::InputArray I_, cv::InputArray J_) {
cv::Mat I = I_.getMat();
cv::Mat J = J_.getMat();

Assertion(I.cols == J.cols && I.rows == J.rows, "Sizes of two images must be the same!");
Assertion(I.depth() == J.depth(), "Bit depths of two images must be the same!");
Assertion((I.channels() == 1 && J.channels() == 1) ||
(I.channels() == 3 && J.channels() == 3),
"Two images must have 1 or 3 channels!");
Assertion((I.depth() == CV_8U && J.depth() == CV_8U) ||
(I.depth() == CV_32F && J.depth() == CV_32F),
"Two images must be with 8-bit unsigned integer or 32-bit floating point number");

if (I.depth() == CV_32F) {
I.convertTo(I, CV_8U, 255.0);
}

if (J.depth() == CV_32F) {
J.convertTo(J, CV_8U, 255.0);
}

// Calculate standard MS-SSIM, first.
const double msssim = MSSSIM(I, J);

// If two images are both grayscale, compute standard MS-SSIM.
if (I.channels() == 1 && J.channels() == 1) {
return msssim;
}

// Convert images to CIE-Lab to evaluate color similarity.
cv::Mat labI, labJ;
I.convertTo(I, CV_32F, 1.0 / 255.0);
J.convertTo(J, CV_32F, 1.0 / 255.0);
cv::GaussianBlur(I, I, cv::Size(5, 5), 0.0);
cv::GaussianBlur(J, J, cv::Size(5, 5), 0.0);
cv::cvtColor(I, labI, cv::COLOR_BGR2Lab);
cv::cvtColor(J, labJ, cv::COLOR_BGR2Lab);

// Calculate color similarity based on Just Noticeable Color Difference (JNCD).
const double delta = 0.7;
const double jncd = 2.3; // 2.3 plus/minus 1.3

const int width = labI.cols;
const int height = labI.rows;
uint64_t count = 0;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
const double diff_x = (double)I.at<uchar>(x, y) - mu_x;
const double diff_y = (double)J.at<uchar>(x, y) - mu_y;
sig_x += diff_x * diff_x;
sig_y += diff_y * diff_y;
cov_xy += diff_x * diff_y;
const cv::Vec3f cI = labI.at<cv::Vec3f>(y, x);
const cv::Vec3f cJ = labJ.at<cv::Vec3f>(y, x);
const double diff = cv::norm(cI - cJ, cv::NORM_L2);
if (diff <= jncd * 3.0) {
count += 1;
}
}
}
sig_x = std::sqrt(sig_x / totalSize);
sig_y = std::sqrt(sig_y / totalSize);
cov_xy /= totalSize;

const double k1 = 0.01;
const double k2 = 0.03;
const double L = 255.0;
const double c1 = (k1 * L) * (k1 * L);
const double c2 = (k2 * L) * (k2 * L);
const double c3 = 0.5 * c2;

const double l = (2.0 * mu_x * mu_y + c1) / (mu_x * mu_x + mu_y * mu_y + c1);
const double c = (2.0 * sig_x * sig_y + c2) / (sig_x * sig_x + sig_y * sig_y + c2);
const double s = (cov_xy + c3) / (sig_x * sig_y + c3);

return l * c * s;
const double clr = (double)count / (double)(width * height);
return std::pow(clr, delta) * msssim;
}

} // namespace lime
Expand Down
27 changes: 21 additions & 6 deletions sources/python/metrics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,28 @@ double py_SSIM(const py::array_t<T>& I, const py::array_t<T>& J) {
return lime::SSIM(np2mat(I), np2mat(J));
}

template <typename T>
double py_MSSSIM(const py::array_t<T>& I, const py::array_t<T>& J) {
return lime::MSSSIM(np2mat(I), np2mat(J));
}

template <typename T>
double py_CMSSIM(const py::array_t<T>& I, const py::array_t<T>& J) {
return lime::CMSSIM(np2mat(I), np2mat(J));
}

void init_metrics(py::module m) {
m.def("MSE", &py_MSE<uint8_t>, "Mean squared error (MSE)");
m.def("MSE", &py_MSE<float>, "Mean squared error (MSE)");
m.def("RMSE", &py_MSE<uint8_t>, "Root mean squared error (RMSE)");
m.def("RMSE", &py_MSE<float>, "Root mean squared error (RMSE)");
m.def("PSNR", &py_MSE<uint8_t>, "Peak signal-to-noise ratio (PSNR)");
m.def("PSNR", &py_MSE<float>, "Peak signal-to-noise ratio (PSNR)");
m.def("SSIM", &py_MSE<uint8_t>, "Structural similarity (SSIM)");
m.def("SSIM", &py_MSE<float>, "Structural similarity (SSIM)");
m.def("RMSE", &py_RMSE<uint8_t>, "Root mean squared error (RMSE)");
m.def("RMSE", &py_RMSE<float>, "Root mean squared error (RMSE)");
m.def("PSNR", &py_PSNR<uint8_t>, "Peak signal-to-noise ratio (PSNR)");
m.def("PSNR", &py_PSNR<float>, "Peak signal-to-noise ratio (PSNR)");

m.def("SSIM", &py_SSIM<uint8_t>, "Structural similarity (SSIM)");
m.def("SSIM", &py_SSIM<float>, "Structural similarity (SSIM)");
m.def("MSSSIM", &py_MSSSIM<uint8_t>, "Multi-scale structural similarity (MS-SSIM)");
m.def("MSSSIM", &py_MSSSIM<float>, "Multi-scale structural similarity (MS-SSIM)");
m.def("CMSSIM", &py_CMSSIM<uint8_t>, "Color multi-scale structural similarity (CM-SSIM)");
m.def("CMSSIM", &py_CMSSIM<float>, "Color multi-scale structural similarity (CM-SSIM)");
}
Loading

0 comments on commit adfe8af

Please sign in to comment.