Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

attempt to add 0d/1d mat support to OpenCV #23473

Merged
merged 25 commits into from Sep 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
b22efac
attempt to add 0d/1d mat support to OpenCV
vpisarev Apr 8, 2023
64a5da1
revised the patch; now 1D mat is treated as 1xN 2D mat rather than Nx1.
vpisarev Jun 8, 2023
60fefa1
a step towards 'green' tests
vpisarev Jun 8, 2023
a8c98e2
another little step towards 'green' tests
vpisarev Jun 9, 2023
d2c8b1e
calib test failures seem to be fixed now
vpisarev Jun 9, 2023
8aaeb26
more fixes _core & _dnn
vpisarev Jun 11, 2023
d2f1c39
another step towards green ci; even 0D mat's (a.k.a. scalars) are now…
vpisarev Jun 13, 2023
901ace8
* fixed strange bug in aruco/charuco detector, not sure why it did no…
vpisarev Jun 13, 2023
ced94fb
disabled failing GAPI tests - too complex to dig into this compiler p…
vpisarev Jun 13, 2023
0a1a28c
hopefully fixed java tests
vpisarev Jun 14, 2023
d8af48e
trying to fix some more tests
vpisarev Jun 14, 2023
32ed0c1
quick followup fix
vpisarev Jun 14, 2023
92d83a3
continue to fix test failures and warnings
vpisarev Jun 14, 2023
9f45ae9
quick followup fix
vpisarev Jun 14, 2023
1075850
trying to fix some more tests
vpisarev Jun 15, 2023
ab60f13
partly fixed support for 0D/scalar UMat's
vpisarev Jun 16, 2023
296c284
use updated parseReduce() from upstream
vpisarev Aug 8, 2023
cf6ed96
trying to fix the remaining test failures
vpisarev Aug 31, 2023
53f3706
fixed [ch]aruco tests in Python
vpisarev Aug 31, 2023
ab40691
still trying to fix tests
vpisarev Sep 1, 2023
5032600
revert "fix" in dnn's CUDA tensor
vpisarev Sep 1, 2023
0e85eee
trying to fix dnn+CUDA test failures
vpisarev Sep 17, 2023
f72c2aa
fixed 1D umat creation
vpisarev Sep 17, 2023
06097dc
hopefully fixed remaining cuda test failures
vpisarev Sep 21, 2023
80c7907
removed training whitespaces
vpisarev Sep 21, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion modules/3d/misc/java/test/Cv3dTest.java
Expand Up @@ -549,7 +549,8 @@ public void testUndistortPointsListOfPointListOfPointMatMat() {

Cv3d.undistortPoints(src, dst, cameraMatrix, distCoeffs);

assertEquals(src.size(), dst.size());
assertEquals(src.cols(), dst.rows());
assertEquals(src.rows(), dst.cols());
Comment on lines 550 to +553
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we want to swap rows/cols?
This doesn't look like a convenient behavior.

for(int i=0; i<src.toList().size(); i++) {
//Log.d("UndistortPoints", "s="+src.get(i)+", d="+dst.get(i));
assertTrue(src.toList().get(i).equals(dst.toList().get(i)));
Expand Down
6 changes: 4 additions & 2 deletions modules/3d/perf/perf_pnp.cpp
Expand Up @@ -40,7 +40,8 @@ PERF_TEST_P(PointsNum_Algo, solvePnP,
projectPoints(points3d, rvec, tvec, intrinsics, distortion, points2d);

//add noise
Mat noise(1, (int)points2d.size(), CV_32FC2);
int sz = (int)points2d.size();
Mat noise(1, &sz, CV_32FC2);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Requirement of test modification means that we would broke user code.
We could add new tests easily, but not to modify it.

We need compatibility code to support both inputs (old and new).

randu(noise, 0, 0.01);
cv::add(points2d, noise, points2d);

Expand Down Expand Up @@ -93,7 +94,8 @@ PERF_TEST_P(PointsNum_Algo, solvePnPSmallPoints,
cv::projectPoints(points3d, rvec, tvec, intrinsics, distortion, points2d);

//add noise
Mat noise(1, (int)points2d.size(), CV_32FC2);
int npoints = (int)points2d.size();
Mat noise(1, &npoints, CV_32FC2);
randu(noise, -0.001, 0.001);
cv::add(points2d, noise, points2d);

Expand Down
5 changes: 3 additions & 2 deletions modules/3d/src/solvepnp.cpp
Expand Up @@ -1112,9 +1112,10 @@ int solvePnPGeneric( InputArray _opoints, InputArray _ipoints,

for (size_t i = 0; i < vec_rvecs.size(); i++)
{
std::vector<Point2d> projectedPoints;
Mat projectedPoints;
projectPoints(objectPoints, vec_rvecs[i], vec_tvecs[i], cameraMatrix, distCoeffs, projectedPoints);
double rmse = norm(Mat(projectedPoints, false), imagePoints, NORM_L2) / sqrt(2*projectedPoints.size());
int nprojectedPoints = (int)projectedPoints.total();
double rmse = norm(projectedPoints, imagePoints, NORM_L2) / sqrt(2*nprojectedPoints);

Mat err = reprojectionError.getMat();
if (type == CV_32F)
Expand Down
8 changes: 4 additions & 4 deletions modules/calib/misc/java/test/CalibTest.java
Expand Up @@ -67,8 +67,8 @@ public void testFindCirclesGridMatSizeMat() {

assertTrue(Calib.findCirclesGrid(img, new Size(5, 5), centers));

assertEquals(25, centers.rows());
assertEquals(1, centers.cols());
assertEquals(1, centers.rows());
assertEquals(25, centers.cols());
assertEquals(CvType.CV_32FC2, centers.type());
}

Expand All @@ -93,8 +93,8 @@ public void testFindCirclesGridMatSizeMatInt() {
assertTrue(Calib.findCirclesGrid(img, new Size(3, 5), centers, Calib.CALIB_CB_CLUSTERING
| Calib.CALIB_CB_ASYMMETRIC_GRID));

assertEquals(15, centers.rows());
assertEquals(1, centers.cols());
assertEquals(1, centers.rows());
assertEquals(15, centers.cols());
assertEquals(CvType.CV_32FC2, centers.type());
}

Expand Down
2 changes: 1 addition & 1 deletion modules/calib/src/calibration.cpp
Expand Up @@ -334,7 +334,7 @@ static double calibrateCameraInternal( const Mat& objectPoints,
//std::cout << "dist0:" << _k << std::endl;

std::vector<double> param(nparams, 0.0);
Mat paramM(param, false);
Mat paramM = Mat(param, false).reshape(1, nparams);
std::vector<uchar> mask(nparams, (uchar)1);

int solveMethod = DECOMP_EIG;
Expand Down
2 changes: 1 addition & 1 deletion modules/calib/src/chessboard.cpp
Expand Up @@ -272,7 +272,7 @@ void polyfit(const Mat& src_x, const Mat& src_y, Mat& dst, int order)
A.at<double>(y,x) = srcX.at<double>(y)*A.at<double>(y,x-1);
}
cv::Mat w;
solve(A,srcY,w,DECOMP_SVD);
solve(A,srcY.reshape(1, npoints),w,DECOMP_SVD);
w.convertTo(dst, ((src_x.depth() == CV_64F || src_y.depth() == CV_64F) ? CV_64F : CV_32F));
}

Expand Down
14 changes: 8 additions & 6 deletions modules/calib/src/circlesgrid.cpp
Expand Up @@ -204,16 +204,16 @@ void CirclesGridClusterFinder::findCorners(const std::vector<cv::Point2f> &hull2
//corners are the most sharp angles (6)
Mat anglesMat = Mat(angles);
Mat sortedIndices;
sortIdx(anglesMat, sortedIndices, SORT_EVERY_COLUMN + SORT_DESCENDING);
sortIdx(anglesMat, sortedIndices, SORT_EVERY_ROW + SORT_DESCENDING);
CV_Assert(sortedIndices.type() == CV_32SC1);
CV_Assert(sortedIndices.cols == 1);
CV_Assert(sortedIndices.rows == 1);
const int cornersCount = isAsymmetricGrid ? 6 : 4;
Mat cornersIndices;
cv::sort(sortedIndices.rowRange(0, cornersCount), cornersIndices, SORT_EVERY_COLUMN + SORT_ASCENDING);
cv::sort(sortedIndices.colRange(0, cornersCount), cornersIndices, SORT_EVERY_ROW + SORT_ASCENDING);
corners.clear();
for(int i=0; i<cornersCount; i++)
{
corners.push_back(hull2f[cornersIndices.at<int>(i, 0)]);
corners.push_back(hull2f[cornersIndices.at<int>(i)]);
}
}

Expand Down Expand Up @@ -427,7 +427,8 @@ void CirclesGridClusterFinder::parsePatternPoints(const std::vector<cv::Point2f>
CV_Error(Error::StsNotImplemented, "The desired functionality requires flann module, which was disabled.");
#else
flann::LinearIndexParams flannIndexParams;
flann::Index flannIndex(Mat(rectifiedPatternPoints).reshape(1), flannIndexParams);
flann::Index flannIndex(Mat(rectifiedPatternPoints).reshape(1,
(int)rectifiedPatternPoints.size()), flannIndexParams);

centers.clear();
for( int i = 0; i < patternSize.height; i++ )
Expand Down Expand Up @@ -1126,7 +1127,8 @@ void CirclesGridFinder::findBasis(const std::vector<Point2f> &samples, std::vect
TermCriteria termCriteria;
Mat centers;
const int clustersCount = 4;
kmeans(Mat(samples).reshape(1, 0), clustersCount, bestLabels, termCriteria, parameters.kmeansAttempts,
int nsamples = (int)samples.size();
kmeans(Mat(samples).reshape(1, nsamples), clustersCount, bestLabels, termCriteria, parameters.kmeansAttempts,
KMEANS_RANDOM_CENTERS, centers);
CV_Assert( centers.type() == CV_32FC1 );

Expand Down
25 changes: 16 additions & 9 deletions modules/calib/src/fisheye.cpp
Expand Up @@ -183,8 +183,8 @@ double cv::fisheye::calibrate(InputArrayOfArrays objectPoints, InputArrayOfArray
}
else
{
if (rvecs.needed()) Mat(omc).convertTo(rvecs, rvecs.empty() ? CV_64FC3 : rvecs.type());
if (tvecs.needed()) Mat(Tc).convertTo(tvecs, tvecs.empty() ? CV_64FC3 : tvecs.type());
if (rvecs.needed()) Mat(omc).reshape(3, (int)omc.size()).convertTo(rvecs, rvecs.empty() ? CV_64FC3 : rvecs.type());
if (tvecs.needed()) Mat(Tc).reshape(3, (int)Tc.size()).convertTo(tvecs, tvecs.empty() ? CV_64FC3 : tvecs.type());
}

return rms;
Expand Down Expand Up @@ -340,7 +340,9 @@ double cv::fisheye::stereoCalibrate(InputArrayOfArrays objectPoints, InputArrayO
Mat rvec = Mat(rvecs1[image_idx]);
Mat tvec = Mat(tvecs1[image_idx]);
cv::internal::projectPoints(object, projected, rvec, tvec, intrinsicLeft, jacobians);
Mat(Mat((imageLeft - projected).t()).reshape(1, 1).t()).copyTo(ekk.rowRange(0, 2 * n_points));
Mat pt_diff = imageLeft.reshape(1, n_points*2) -
projected.reshape(1, n_points*2);
pt_diff.copyTo(ekk.rowRange(0, 2 * n_points));
jacobians.colRange(8, 11).copyTo(Jkk.colRange(24 + image_idx * 6, 27 + image_idx * 6).rowRange(0, 2 * n_points));
jacobians.colRange(11, 14).copyTo(Jkk.colRange(27 + image_idx * 6, 30 + image_idx * 6).rowRange(0, 2 * n_points));
jacobians.colRange(0, 2).copyTo(Jkk.colRange(0, 2).rowRange(0, 2 * n_points));
Expand All @@ -354,7 +356,10 @@ double cv::fisheye::stereoCalibrate(InputArrayOfArrays objectPoints, InputArrayO
tvec = Mat(tvecs2[image_idx]);

cv::internal::projectPoints(object, projected, omr, Tr, intrinsicRight, jacobians);
Mat(Mat((imageRight - projected).t()).reshape(1, 1).t()).copyTo(ekk.rowRange(2 * n_points, 4 * n_points));

pt_diff = imageRight.reshape(1, n_points*2) -
projected.reshape(1, n_points*2);
pt_diff.copyTo(ekk.rowRange(2 * n_points, 4 * n_points));
Mat dxrdom = jacobians.colRange(8, 11) * domrdom + jacobians.colRange(11, 14) * dTrdom;
Mat dxrdT = jacobians.colRange(8, 11) * domrdT + jacobians.colRange(11, 14)* dTrdT;
Mat dxrdomckk = jacobians.colRange(8, 11) * domrdomckk + jacobians.colRange(11, 14) * dTrdomckk;
Expand Down Expand Up @@ -580,7 +585,7 @@ void cv::internal::ComputeExtrinsicRefine(const Mat& imagePoints, const Mat& obj
Mat jacobians;
projectPoints(objectPoints, x, rvec, tvec, param, jacobians);

Mat ex = imagePoints - Mat(x).t();
Mat ex = imagePoints - Mat(x);
ex = ex.reshape(1, 2);

J = jacobians.colRange(8, 14).clone();
Expand Down Expand Up @@ -826,7 +831,7 @@ void cv::internal::ComputeJacobians(InputArrayOfArrays objectPoints, InputArrayO
objectPoints.getMat(image_idx).convertTo(object, CV_64FC3);
imagePoints.getMat (image_idx).convertTo(image, CV_64FC2);

bool imT = image.rows < image.cols;
bool imT = image.channels() == 1 && image.rows > image.cols;
Mat om(omc.getMat().col(image_idx)), T(Tc.getMat().col(image_idx));

std::vector<Point2d> x;
Expand All @@ -850,8 +855,9 @@ void cv::internal::ComputeJacobians(InputArrayOfArrays objectPoints, InputArrayO
JJ2(Rect(9 + 6 * image_idx, 0, 6, 9)) = A * B.t();
JJ2(Rect(0, 9 + 6 * image_idx, 9, 6)) = JJ2(Rect(9 + 6 * image_idx, 0, 6, 9)).t();

ex3.rowRange(0, 9) += A * exkk.reshape(1, 2 * exkk.rows);
ex3.rowRange(9 + 6 * image_idx, 9 + 6 * (image_idx + 1)) = B * exkk.reshape(1, 2 * exkk.rows);
Mat exkk_col = exkk.reshape(1, 2 * (int)exkk.total());
ex3.rowRange(0, 9) += A * exkk_col;
ex3.rowRange(9 + 6 * image_idx, 9 + 6 * (image_idx + 1)) = B * exkk_col;

if (check_cond)
{
Expand Down Expand Up @@ -891,13 +897,14 @@ void cv::internal::EstimateUncertainties(InputArrayOfArrays objectPoints, InputA
objectPoints.getMat(image_idx).convertTo(object, CV_64FC3);
imagePoints.getMat (image_idx).convertTo(image, CV_64FC2);

bool imT = image.rows < image.cols;
bool imT = image.channels() == 1 && image.rows > image.cols;

Mat om(omc.getMat().col(image_idx)), T(Tc.getMat().col(image_idx));

std::vector<Point2d> x;
projectPoints(object, x, om, T, params, noArray());
Mat ex_ = (imT ? image.t() : image) - Mat(x);
ex_ = ex_.reshape(2, (int)ex_.total());
ex_.copyTo(ex.rowRange(insert_idx, insert_idx + ex_.rows));
insert_idx += ex_.rows;
}
Expand Down
2 changes: 1 addition & 1 deletion modules/calib/test/test_fisheye.cpp
Expand Up @@ -206,7 +206,7 @@ TEST_F(fisheyeTest, Homography)
cv::Mat _objectPoints(objectPoints[0]);

cv::Mat imagePointsNormalized = NormalizePixels(_imagePoints, param).reshape(1).t();
_objectPoints = _objectPoints.reshape(1).t();
_objectPoints = _objectPoints.reshape(1, (int)_objectPoints.total()).t();
cv::Mat objectPointsMean, covObjectPoints;

int Np = imagePointsNormalized.cols;
Expand Down
2 changes: 1 addition & 1 deletion modules/calib/test/test_multiview_calib.cpp
Expand Up @@ -58,7 +58,7 @@ TEST(multiview_calibration, accuracy) {

const int MAX_SAMPLES = 2000, MAX_FRAMES = 50;
cv::Mat pattern (board_pattern, true/*copy*/);
pattern = pattern.reshape(1).t();
pattern = pattern.reshape(1, num_pts).t();
pattern.row(2) = 2.0; // set approximate depth of object points
const double ty_min = -2, ty_max = 2, tx_min = -2, tx_max = 2, tz_min = -1, tz_max = 1;
const double yaw_min = -45, yaw_max = 45, pitch_min = -45, pitch_max = 45, roll_min = -45, roll_max = 45;
Expand Down
23 changes: 21 additions & 2 deletions modules/core/include/opencv2/core/mat.hpp
Expand Up @@ -243,6 +243,7 @@ class CV_EXPORTS _InputArray
bool isUMat() const;
bool isMatVector() const;
bool isUMatVector() const;
bool isVecVector() const;
bool isMatx() const;
bool isVector() const;
bool isGpuMat() const;
Expand Down Expand Up @@ -355,6 +356,9 @@ class CV_EXPORTS _OutputArray : public _InputArray
UMat& getUMatRef(int i=-1) const;
cuda::GpuMat& getGpuMatRef() const;
std::vector<cuda::GpuMat>& getGpuMatVecRef() const;
std::vector<Mat>& getMatVecRef() const;
std::vector<UMat>& getUMatVecRef() const;
template<typename _Tp> std::vector<std::vector<_Tp> >& getVecVecRef() const;
ogl::Buffer& getOGlBufferRef() const;
cuda::HostMem& getHostMemRef() const;
void create(Size sz, int type, int i=-1, bool allowTransposed=false, _OutputArray::DepthMask fixedDepthMask=static_cast<_OutputArray::DepthMask>(0)) const;
Expand Down Expand Up @@ -602,7 +606,7 @@ struct CV_EXPORTS MatStep
MatStep& operator = (size_t s);

size_t* p;
size_t buf[2];
size_t buf[3];
protected:
MatStep& operator = (const MatStep&);
};
Expand Down Expand Up @@ -1510,6 +1514,15 @@ class CV_EXPORTS Mat
*/
void create(const std::vector<int>& sizes, int type);

/** @brief Creates the matrix of the same size as another array.

The method is similar to _OutputArray::createSameSize(arr, type),
but is applied to Mat.
@param arr The other array.
@param type New matrix type.
*/
void createSameSize(InputArray arr, int type);

/** @brief Increments the reference counter.

The method increments the reference counter associated with the matrix data. If the matrix header
Expand Down Expand Up @@ -2134,6 +2147,7 @@ class CV_EXPORTS Mat
int dims;
//! the number of rows and columns or (-1, -1) when the matrix has more than 2 dimensions
int rows, cols;
int dummy = 153;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will remove it in one of the subsequent commits in a different pull request;
Let it stay here for a few weeks/months to catch the situation when Mat.size.p (when it points to rows or cols) is accessed outside of those 2 fields

//! pointer to the data
uchar* data;

Expand Down Expand Up @@ -2307,6 +2321,8 @@ template<typename _Tp> class Mat_ : public Mat
void create(Size _size);
//! equivalent to Mat::create(_ndims, _sizes, DatType<_Tp>::type)
void create(int _ndims, const int* _sizes);
//! equivalent to Mat::create(arr.ndims, arr.size.p, DatType<_Tp>::type)
void createSameSize(InputArray arr);
//! equivalent to Mat::release()
void release();
//! cross-product
Expand Down Expand Up @@ -2526,6 +2542,10 @@ class CV_EXPORTS UMat
void create(int ndims, const int* sizes, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);
void create(const std::vector<int>& sizes, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);

//! allocates new matrix data unless the matrix already has specified size and type.
// the size is taken from the specified array.
void createSameSize(InputArray arr, int type, UMatUsageFlags usageFlags = USAGE_DEFAULT);

//! increases the reference counter; use with care to avoid memleaks
void addref();
//! decreases reference counter;
Expand Down Expand Up @@ -2980,7 +3000,6 @@ class CV_EXPORTS SparseMat
};



///////////////////////////////// SparseMat_<_Tp> ////////////////////////////////////

/** @brief Template sparse n-dimensional array class derived from SparseMat
Expand Down