Skip to content

Commit

Permalink
Merge pull request #3351 from bit1002lst:decode_multi_qrcodes
Browse files Browse the repository at this point in the history
fix the bug that cannot detect multi qrcode when use_nn_detector is false
  • Loading branch information
asmorkalov committed Sep 28, 2022
2 parents 94d15ee + a3afb87 commit cef9724
Show file tree
Hide file tree
Showing 10 changed files with 255 additions and 55 deletions.
67 changes: 67 additions & 0 deletions modules/wechat_qrcode/samples/qrcode_example_without_nn.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#include <iostream>
#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>

using namespace std;
using namespace cv;

#include <opencv2/wechat_qrcode.hpp>

int main(int argc, char* argv[]) {
cout << endl << argv[0] << endl << endl;
cout << "A demo program of WeChat QRCode Detector: " << endl;

Mat img;
int camIdx = -1;
if (argc > 1) {
bool live = strcmp(argv[1], "-camera") == 0;
if (live) {
camIdx = argc > 2 ? atoi(argv[2]) : 0;
} else {
img = imread(argv[1]);
}
} else {
cout << " Usage: " << argv[0] << " <input_image>" << endl;
return 0;
}
// The model is downloaded to ${CMAKE_BINARY_DIR}/downloads/wechat_qrcode if cmake runs without warnings,
// otherwise you can download them from https://github.com/WeChatCV/opencv_3rdparty/tree/wechat_qrcode.
Ptr<wechat_qrcode::WeChatQRCode> detector;

try {
detector = makePtr<wechat_qrcode::WeChatQRCode>("", "", "", "");
} catch (const std::exception& e) {
cout <<
"\n---------------------------------------------------------------\n"
"Failed to initialize WeChatQRCode.\n"
"---------------------------------------------------------------\n";
cout << e.what() << endl;
return 0;
}
string prevstr = "";
vector<Mat> points;

if (camIdx < 0) {
auto res = detector->detectAndDecode(img, points);
for (const auto& t : res) cout << t << endl;
} else {
VideoCapture cap(camIdx);
for(;;) {
cap >> img;
if (img.empty())
break;
auto res = detector->detectAndDecode(img, points);
for (const auto& t : res) {
if (t != prevstr)
cout << t << endl;
}
if (!res.empty())
prevstr = res.back();
imshow("image", img);
if (waitKey(30) >= 0)
break;
}
}
return 0;
}
50 changes: 50 additions & 0 deletions modules/wechat_qrcode/samples/qrcode_without_nn.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import cv2
import sys

print(sys.argv[0])
print('A demo program of WeChat QRCode Detector:')
camIdx = -1
if len(sys.argv) > 1:
if sys.argv[1] == "-camera":
camIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0
img = cv2.imread(sys.argv[1])
else:
print(" Usage: " + sys.argv[0] + " <input_image>")
exit(0)

# For python API generator, it follows the template: {module_name}_{class_name},
# so it is a little weird.
# The model is downloaded to ${CMAKE_BINARY_DIR}/downloads/wechat_qrcode if cmake runs without warnings,
# otherwise you can download them from https://github.com/WeChatCV/opencv_3rdparty/tree/wechat_qrcode.
try:
detector = cv2.wechat_qrcode_WeChatQRCode(
"", "", "", "")
except:
print("---------------------------------------------------------------")
print("Failed to initialize WeChatQRCode.")
print("---------------------------------------------------------------")
exit(0)

prevstr = ""

if camIdx < 0:
res, points = detector.detectAndDecode(img)
print(res,points)
else:
cap = cv2.VideoCapture(camIdx)
while True:
res, img = cap.read()
if img is None:
break
res, points = detector.detectAndDecode(img)
for t in res:
if t != prevstr:
print(t)
if res:
prevstr = res[-1]
cv2.imshow("image", img)
if cv2.waitKey(30) >= 0:
break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
36 changes: 20 additions & 16 deletions modules/wechat_qrcode/src/decodermgr.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ using zxing::Result;
using zxing::UnicomBlock;
namespace cv {
namespace wechat_qrcode {
int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result, vector<Point2f>& points) {
int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, vector<string>& results, vector<vector<Point2f>>& zxing_points) {
int width = src.cols;
int height = src.rows;
if (width <= 20 || height <= 20)
Expand All @@ -28,7 +28,7 @@ int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result, v
zxing::ArrayRef<uint8_t> scaled_img_zx =
zxing::ArrayRef<uint8_t>(new zxing::Array<uint8_t>(scaled_img_data));

zxing::Ref<zxing::Result> zx_result;
vector<zxing::Ref<zxing::Result>> zx_results;

decode_hints_.setUseNNDetector(use_nn_detector);

Expand All @@ -43,16 +43,20 @@ int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result, v
} else {
source->reset(scaled_img_zx.data(), width, height);
}
int ret = TryDecode(source, zx_result);
int ret = TryDecode(source, zx_results);
if (!ret) {
result = zx_result->getText()->getText();
auto result_points = zx_result->getResultPoints();
for(int i = 0; i < result_points->size() / 4; i++) {
const int ind = i * 4;
for (int j = 1; j < 4; j++)
points.emplace_back(result_points[ind+j]->getX(), result_points[ind+j]->getY());

points.emplace_back(result_points[ind]->getX(), result_points[ind]->getY());
for(size_t k = 0; k < zx_results.size(); k++) {
results.emplace_back(zx_results[k]->getText()->getText());
vector<Point2f> tmp_qr_points;
auto tmp_zx_points = zx_results[k]->getResultPoints();
for (int i = 0; i < tmp_zx_points->size() / 4; i++) {
const int ind = i * 4;
for (int j = 1; j < 4; j++){
tmp_qr_points.emplace_back(tmp_zx_points[ind + j]->getX(), tmp_zx_points[ind + j]->getY());
}
tmp_qr_points.emplace_back(tmp_zx_points[ind]->getX(), tmp_zx_points[ind]->getY());
}
zxing_points.push_back(tmp_qr_points);
}
return ret;
}
Expand All @@ -62,7 +66,7 @@ int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result, v
return -1;
}

int DecoderMgr::TryDecode(Ref<LuminanceSource> source, Ref<Result>& result) {
int DecoderMgr::TryDecode(Ref<LuminanceSource> source, vector<Ref<Result>>& results) {
int res = -1;
string cell_result;

Expand All @@ -71,17 +75,17 @@ int DecoderMgr::TryDecode(Ref<LuminanceSource> source, Ref<Result>& result) {
zxing::Ref<zxing::BinaryBitmap> binary_bitmap(new BinaryBitmap(binarizer));
binary_bitmap->m_poUnicomBlock = qbarUicomBlock_;

result = Decode(binary_bitmap, decode_hints_);
res = (result == NULL) ? 1 : 0;
results = Decode(binary_bitmap, decode_hints_);
res = (results.size() == 0) ? 1 : 0;

if (res == 0) {
result->setBinaryMethod(int(binarizer_mgr_.GetCurBinarizer()));
results[0]->setBinaryMethod(int(binarizer_mgr_.GetCurBinarizer()));
}

return res;
}

Ref<Result> DecoderMgr::Decode(Ref<BinaryBitmap> image, DecodeHints hints) {
vector<Ref<Result>> DecoderMgr::Decode(Ref<BinaryBitmap> image, DecodeHints hints) {
return reader_->decode(image, hints);
}
} // namespace wechat_qrcode
Expand Down
6 changes: 3 additions & 3 deletions modules/wechat_qrcode/src/decodermgr.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class DecoderMgr {
DecoderMgr() { reader_ = new zxing::qrcode::QRCodeReader(); };
~DecoderMgr(){};

int decodeImage(cv::Mat src, bool use_nn_detector, string& result, vector<Point2f>& points);
int decodeImage(cv::Mat src, bool use_nn_detector, vector<string>& result, vector<vector<Point2f>>& zxing_points);

private:
zxing::Ref<zxing::UnicomBlock> qbarUicomBlock_;
Expand All @@ -35,10 +35,10 @@ class DecoderMgr {
zxing::Ref<zxing::qrcode::QRCodeReader> reader_;
BinarizerMgr binarizer_mgr_;

zxing::Ref<zxing::Result> Decode(zxing::Ref<zxing::BinaryBitmap> image,
vector<zxing::Ref<zxing::Result>> Decode(zxing::Ref<zxing::BinaryBitmap> image,
zxing::DecodeHints hints);

int TryDecode(zxing::Ref<zxing::LuminanceSource> source, zxing::Ref<zxing::Result>& result);
int TryDecode(zxing::Ref<zxing::LuminanceSource> source, vector<zxing::Ref<zxing::Result>>& result);
};

} // namespace wechat_qrcode
Expand Down
49 changes: 36 additions & 13 deletions modules/wechat_qrcode/src/wechat_qrcode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -144,21 +144,44 @@ vector<string> WeChatQRCode::Impl::decode(const Mat& img, vector<Mat>& candidate
super_resolution_model_->processImageScale(cropped_img, cur_scale, use_nn_sr_);
string result;
DecoderMgr decodemgr;
vector<Point2f> points_qr;
auto ret = decodemgr.decodeImage(scaled_img, use_nn_detector_, result, points_qr);
vector<vector<Point2f>> zxing_points, check_points;
auto ret = decodemgr.decodeImage(scaled_img, use_nn_detector_, decode_results, zxing_points);
if (ret == 0) {
for (auto&& pt: points_qr) {
pt /= cur_scale;
for(size_t i = 0; i <zxing_points.size(); i++){
vector<Point2f> points_qr = zxing_points[i];
for (auto&& pt: points_qr) {
pt /= cur_scale;
}

if (use_nn_detector_)
points_qr = aligner.warpBack(points_qr);
for (int j = 0; j < 4; ++j) {
point.at<float>(j, 0) = points_qr[j].x;
point.at<float>(j, 1) = points_qr[j].y;
}
// try to find duplicate qr corners
bool isDuplicate = false;
for (const auto &tmp_points: check_points) {
const float eps = 10.f;
for (size_t j = 0; j < tmp_points.size(); j++) {
if (abs(tmp_points[j].x - points_qr[j].x) < eps &&
abs(tmp_points[j].y - points_qr[j].y) < eps) {
isDuplicate = true;
}
else {
isDuplicate = false;
break;
}
}
}
if (isDuplicate == false) {
points.push_back(point);
check_points.push_back(points_qr);
}
else {
decode_results.erase(decode_results.begin() + i, decode_results.begin() + i + 1);
}
}

if (use_nn_detector_)
points_qr = aligner.warpBack(points_qr);
for (int i = 0; i < 4; ++i) {
point.at<float>(i, 0) = points_qr[i].x;
point.at<float>(i, 1) = points_qr[i].y;
}
decode_results.push_back(result);
points.push_back(point);
break;
}
}
Expand Down
43 changes: 26 additions & 17 deletions modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,33 +34,33 @@ QRCodeReader::QRCodeReader() : decoder_() {
smoothMaxMultiple_ = 40;
}

Ref<Result> QRCodeReader::decode(Ref<BinaryBitmap> image) { return decode(image, DecodeHints()); }
vector<Ref<Result>> QRCodeReader::decode(Ref<BinaryBitmap> image) { return decode(image, DecodeHints()); }

Ref<Result> QRCodeReader::decode(Ref<BinaryBitmap> image, DecodeHints hints) {
vector<Ref<Result>> QRCodeReader::decode(Ref<BinaryBitmap> image, DecodeHints hints) {
// Binarize image using the Histogram Binarized method and be binarized
ErrorHandler err_handler;
vector<Ref<Result>> result_list;
Ref<BitMatrix> imageBitMatrix = image->getBlackMatrix(err_handler);
if (err_handler.ErrCode() || imageBitMatrix == NULL) return Ref<Result>();
if (err_handler.ErrCode() || imageBitMatrix == NULL) return result_list;

Ref<Result> rst = decodeMore(image, imageBitMatrix, hints, err_handler);
if (err_handler.ErrCode() || rst == NULL) {
vector<Ref<Result>> rst = decodeMore(image, imageBitMatrix, hints, err_handler);
if (err_handler.ErrCode() || rst.empty()) {
// black white mirro!!!
Ref<BitMatrix> invertedMatrix = image->getInvertedMatrix(err_handler);
if (err_handler.ErrCode() || invertedMatrix == NULL) return Ref<Result>();
Ref<Result> tmp_rst = decodeMore(image, invertedMatrix, hints, err_handler);
if (err_handler.ErrCode() || tmp_rst == NULL) return Ref<Result>();
return tmp_rst;
if (err_handler.ErrCode() || invertedMatrix == NULL) return result_list;
vector<Ref<Result>> tmp_rst = decodeMore(image, invertedMatrix, hints, err_handler);
if (err_handler.ErrCode() || tmp_rst.empty()) return tmp_rst;
}

return rst;
}

Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> imageBitMatrix,
vector<Ref<Result>> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> imageBitMatrix,
DecodeHints hints, ErrorHandler &err_handler) {
nowHints_ = hints;
std::string ept;

if (imageBitMatrix == NULL) return Ref<Result>();
vector<Ref<Result>> result_list;
if (imageBitMatrix == NULL) return result_list;
image->m_poUnicomBlock->Init();
image->m_poUnicomBlock->Reset(imageBitMatrix);

Expand Down Expand Up @@ -88,7 +88,7 @@ Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> ima
Ref<FinderPatternInfo> patternInfo = detector->getFinderPatternInfo(i);
setPatternFix(patternInfo->getPossibleFix());
if (patternInfo->getAnglePossibleFix() < 0.6 && i) continue;

bool patternFoundFlag = false;
int possibleAlignmentCount = 0;
possibleAlignmentCount = detector->getPossibleAlignmentCount(i);
if (possibleAlignmentCount < 0) continue;
Expand All @@ -99,6 +99,7 @@ Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> ima

vector<bool> needTryVariousDeimensions(possibleAlignmentCount, false);
for (int j = 0; j < possibleAlignmentCount; j++) {
if (patternFoundFlag){break;}
ArrayRef<Ref<ResultPoint> > points;
err_handler.Reset();
Ref<DetectorResult> detectorResult =
Expand Down Expand Up @@ -142,11 +143,15 @@ Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> ima
decoderResult->getCharset(), decoderResult->getQRCodeVersion(),
decoderResult->getEcLevel(), decoderResult->getCharsetMode()));
setSuccFix(points);

return result;
result_list.push_back(result);
patternFoundFlag = true;
if (nowHints_.getUseNNDetector()) {
return result_list;
}
}
// try different dimentions
for (int j = 0; j < possibleAlignmentCount; j++) {
if (patternFoundFlag){break;}
err_handler.Reset();
ArrayRef<Ref<ResultPoint> > points;
if (needTryVariousDeimensions[j]) {
Expand Down Expand Up @@ -188,13 +193,17 @@ Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> ima
decoderResult->getEcLevel(), decoderResult->getCharsetMode()));

setSuccFix(points);
return result;
result_list.push_back(result);
patternFoundFlag = true;
if (nowHints_.getUseNNDetector()) {
return result_list;
}
}
}
}
}
}
return Ref<Result>();
return result_list;
}

vector<int> QRCodeReader::getPossibleDimentions(int detectDimension) {
Expand Down
6 changes: 3 additions & 3 deletions modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,10 @@ class QRCodeReader : public Reader {
virtual ~QRCodeReader();
string name() override { return "qrcode"; }

Ref<Result> decode(Ref<BinaryBitmap> image) override;
Ref<Result> decode(Ref<BinaryBitmap> image, DecodeHints hints) override;
vector<Ref<Result>> decode(Ref<BinaryBitmap> image) override;
vector<Ref<Result>> decode(Ref<BinaryBitmap> image, DecodeHints hints) override;

Ref<Result> decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> imageBitMatrix,
vector<Ref<Result>> decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> imageBitMatrix,
DecodeHints hints, ErrorHandler& err_handler);

private:
Expand Down
Loading

0 comments on commit cef9724

Please sign in to comment.