diff --git a/airtest/aircv/error.py b/airtest/aircv/error.py index e878d4f6..29f59f84 100644 --- a/airtest/aircv/error.py +++ b/airtest/aircv/error.py @@ -51,3 +51,7 @@ class NoMatchPointError(BaseError): class MatchResultCheckError(BaseError): """Exception raised for errors 0 keypoint found in the input images.""" + + +class PerspectiveTransformError(BaseError): + """ An error occurred while perspectiveTransform """ diff --git a/airtest/aircv/keypoint_base.py b/airtest/aircv/keypoint_base.py index 8aaf4552..afe6d788 100644 --- a/airtest/aircv/keypoint_base.py +++ b/airtest/aircv/keypoint_base.py @@ -6,14 +6,18 @@ import cv2 import time import numpy as np +from distutils.version import LooseVersion from airtest.utils.logger import get_logger from .error import * # noqa -from .utils import generate_result, check_image_valid, print_run_time +from .utils import (generate_result, check_image_valid, print_run_time, get_keypoint_from_matches, rectangle_transform, + keypoint_distance, get_middle_point) from .cal_confidence import cal_ccoeff_confidence, cal_rgb_confidence - LOGGING = get_logger(__name__) +CVRANSAC = (cv2.RANSAC, 5.0) +if LooseVersion(cv2.__version__) > LooseVersion('4.5.0'): + CVRANSAC = (cv2.USAC_MAGSAC, 4.0, None, 2000, 0.99) class KeypointMatching(object): @@ -38,128 +42,178 @@ def mask_kaze(self): # 求出特征点后，self.im_source中获得match的那些点进行聚类 raise NotImplementedError - def find_all_results(self): - """基于kaze查找多个目标区域的方法.""" - # 求出特征点后，self.im_source中获得match的那些点进行聚类 - raise NotImplementedError - @print_run_time - def find_best_result(self): - """基于kaze进行图像识别，只筛选出最优区域.""" - # 第一步：检验图像是否正常： + def find_all_results(self, max_count=10, max_iter_counts=20, distance_threshold=150): if not check_image_valid(self.im_source, self.im_search): return None + result = [] - # 第二步：获取特征点集并匹配出特征点对: 返回值 good, pypts, kp_sch, kp_src - self.kp_sch, self.kp_src, self.good = self._get_key_points() - - # 第三步：根据匹配点对(good),提取出来识别区域: - if len(self.good) in [0, 1]: - # 匹配点对为0,无法提取识别区域;为1则无法获取目标区域,直接返回None作为匹配结果: - return None - elif len(self.good) in [2, 3]: - # 匹配点对为2或3,根据点对求出目标区域,据此算出可信度: - if len(self.good) == 2: - origin_result = self._handle_two_good_points(self.kp_sch, self.kp_src, self.good) - else: - origin_result = self._handle_three_good_points(self.kp_sch, self.kp_src, self.good) - # 某些特殊情况下直接返回None作为匹配结果: - if origin_result is None: - return origin_result - else: - middle_point, pypts, w_h_range = origin_result - else: - # 匹配点对 >= 4个，使用单矩阵映射求出目标区域，据此算出可信度： - middle_point, pypts, w_h_range = self._many_good_pts(self.kp_sch, self.kp_src, self.good) - - # 第四步：根据识别区域，求出结果可信度，并将结果进行返回: - # 对识别结果进行合理性校验: 小于5个像素的，或者缩放超过5倍的，一律视为不合法直接raise. - self._target_error_check(w_h_range) - # 将截图和识别结果缩放到大小一致,准备计算可信度 - x_min, x_max, y_min, y_max, w, h = w_h_range - target_img = self.im_source[y_min:y_max, x_min:x_max] - resize_img = cv2.resize(target_img, (w, h)) - confidence = self._cal_confidence(resize_img) - - best_match = generate_result(middle_point, pypts, confidence) - LOGGING.debug("[%s] threshold=%s, result=%s" % (self.METHOD_NAME, self.threshold, best_match)) - return best_match if confidence >= self.threshold else None - - def show_match_image(self): - """Show how the keypoints matches.""" - from random import random - h_sch, w_sch = self.im_search.shape[:2] - h_src, w_src = self.im_source.shape[:2] - - # first you have to do the matching - self.find_best_result() - # then initialize the result image: - matching_info_img = np.zeros([max(h_sch, h_src), w_sch + w_src, 3], np.uint8) - matching_info_img[:h_sch, :w_sch, :] = self.im_search - matching_info_img[:h_src, w_sch:, :] = self.im_source - # render the match image at last: - for m in self.good: - color = tuple([int(random() * 255) for _ in range(3)]) - cv2.line(matching_info_img, (int(self.kp_sch[m.queryIdx].pt[0]), int(self.kp_sch[m.queryIdx].pt[1])), (int(self.kp_src[m.trainIdx].pt[0] + w_sch), int(self.kp_src[m.trainIdx].pt[1])), color) - - return matching_info_img - - def _cal_confidence(self, resize_img): - """计算confidence.""" - if self.rgb: - confidence = cal_rgb_confidence(resize_img, self.im_search) - else: - confidence = cal_ccoeff_confidence(resize_img, self.im_search) - # confidence修正 - confidence = (1 + confidence) / 2 - return confidence - - def init_detector(self): - """Init keypoint detector object.""" - self.detector = cv2.KAZE_create() - # create BFMatcher object: - self.matcher = cv2.BFMatcher(cv2.NORM_L1) # cv2.NORM_L1 cv2.NORM_L2 cv2.NORM_HAMMING(not useable) - - def get_keypoints_and_descriptors(self, image): - """获取图像特征点和描述符.""" - keypoints, descriptors = self.detector.detectAndCompute(image, None) - return keypoints, descriptors - - def match_keypoints(self, des_sch, des_src): - """Match descriptors (特征值匹配).""" - # 匹配两个图片中的特征点集，k=2表示每个特征点取出2个最匹配的对应点: - return self.matcher.knnMatch(des_sch, des_src, k=2) - - def _get_key_points(self): - """根据传入图像,计算图像所有的特征点,并得到匹配特征点对.""" - # 准备工作: 初始化算子 self.init_detector() - # 第一步：获取特征点集，并匹配出特征点对: 返回值 good, pypts, kp_sch, kp_src + kp_sch, des_sch = self.get_keypoints_and_descriptors(self.im_search) kp_src, des_src = self.get_keypoints_and_descriptors(self.im_source) - # When apply knnmatch , make sure that number of features in both test and - # query image is greater than or equal to number of nearest neighbors in knn match. - if len(kp_sch) < 2 or len(kp_src) < 2: - raise NoMatchPointError("Not enough feature points in input images !") - # match descriptors (特征值匹配) - matches = self.match_keypoints(des_sch, des_src) - - # good为特征点初选结果，剔除掉前两名匹配太接近的特征点，不是独特优秀的特征点直接筛除(多目标识别情况直接不适用) - good = [] - for m, n in matches: - if m.distance < self.FILTER_RATIO * n.distance: - good.append(m) - # good点需要去除重复的部分，（设定源图像不能有重复点）去重时将src图像中的重复点找出即可 - # 去重策略：允许搜索图像对源图像的特征点映射一对多，不允许多对一重复（即不能源图像上一个点对应搜索图像的多个点） - good_diff, diff_good_point = [], [[]] - for m in good: - diff_point = [int(kp_src[m.trainIdx].pt[0]), int(kp_src[m.trainIdx].pt[1])] - if diff_point not in diff_good_point: - good_diff.append(m) - diff_good_point.append(diff_point) - good = good_diff - - return kp_sch, kp_src, good + + kp_src, kp_sch = list(kp_src), list(kp_sch) + + match_knn_count = max_count == 1 and 2 or max_count + 2 + + matches = np.array(self.match_keypoints(des_sch, des_src, match_knn_count)) + kp_sch_point = np.array([(kp.pt[0], kp.pt[1], kp.angle) for kp in kp_sch]) + kp_src_matches_point = np.array([[(*kp_src[dMatch.trainIdx].pt, kp_src[dMatch.trainIdx].angle) + if dMatch else np.nan for dMatch in match] for match in matches]) + _max_iter_counts = 0 + while True: + if (np.count_nonzero(~np.isnan(kp_src_matches_point)) == 0) or \ + (len(result) == max_count) or (_max_iter_counts >= max_iter_counts): + break + _max_iter_counts += 1 + + filtered_good_point, angle, first_point = self.filter_good_point(matches=matches, kp_src=kp_src, + kp_sch=kp_sch, + kp_sch_point=kp_sch_point, + kp_src_matches_point=kp_src_matches_point) + # img = cv2.drawMatches(self.im_search, kp_sch, self.im_source, kp_src, filtered_good_point, + # None, matchesThickness=2) + # cv2.imshow('match', img) + if first_point.distance > distance_threshold: + break + pypts, w_h_range, confidence = None, None, 0 + + try: + pypts, w_h_range, confidence = self.extract_good_points(kp_src=kp_src, kp_sch=kp_sch, + good=filtered_good_point, angle=angle) + except (PerspectiveTransformError, cv2.error): + pass + finally: + if w_h_range and confidence >= self.threshold: + # 移除改范围内的所有特征点 ??有可能因为透视变换的原因，删除了多余的特征点 + for index, match in enumerate(kp_src_matches_point): + x, y = match[:, 0], match[:, 1] + flag = np.argwhere((x < w_h_range[1]) & (x > w_h_range[0]) & + (y < w_h_range[3]) & (y > w_h_range[2])) + for _index in flag: + kp_src_matches_point[index, _index, :] = np.nan + matches[index, _index] = np.nan + middle_point = get_middle_point(w_h_range) + result.append(generate_result(middle_point, pypts, confidence)) + else: + for match in filtered_good_point: + flags = np.argwhere(matches[match.queryIdx, :] == match) + for _index in flags: + kp_src_matches_point[match.queryIdx, _index, :] = np.nan + matches[match.queryIdx, _index] = np.nan + + return result + + def find_best_result(self, *args, **kwargs): + ret = self.find_all_results(max_count=1, *args, **kwargs) + + if ret: + best_match = ret[0] + LOGGING.debug("[%s] threshold=%s, result=%s" % (self.METHOD_NAME, self.threshold, best_match)) + return best_match + return None + + @staticmethod + def filter_good_point(matches, kp_src, kp_sch, kp_sch_point, kp_src_matches_point): + """ 筛选最佳点 """ + # 假设第一个点,及distance最小的点,为基准点 + sort_list = [sorted(match, key=lambda x: x is np.nan and float('inf') or x.distance)[0] + for match in matches] + sort_list = [v for v in sort_list if v is not np.nan] + + first_good_point = sorted(sort_list, key=lambda x: x.distance)[0] + first_good_point_train = kp_src[first_good_point.trainIdx] + first_good_point_query = kp_sch[first_good_point.queryIdx] + first_good_point_angle = first_good_point_train.angle - first_good_point_query.angle + + def get_points_origin_angle(point_x, point_y, offset): + points_origin_angle = np.arctan2( + (point_y - offset.pt[1]), + (point_x - offset.pt[0]) + ) * 180 / np.pi + + points_origin_angle = np.where( + points_origin_angle == 0, + points_origin_angle, points_origin_angle - offset.angle + ) + points_origin_angle = np.where( + points_origin_angle >= 0, + points_origin_angle, points_origin_angle + 360 + ) + return points_origin_angle + + # 计算模板图像上,该点与其他特征点的旋转角 + first_good_point_sch_origin_angle = get_points_origin_angle(kp_sch_point[:, 0], kp_sch_point[:, 1], + first_good_point_query) + + # 计算目标图像中,该点与其他特征点的夹角 + kp_sch_rotate_angle = kp_sch_point[:, 2] + first_good_point_angle + kp_sch_rotate_angle = np.where(kp_sch_rotate_angle >= 360, kp_sch_rotate_angle - 360, kp_sch_rotate_angle) + kp_sch_rotate_angle = kp_sch_rotate_angle.reshape(kp_sch_rotate_angle.shape + (1,)) + + kp_src_angle = kp_src_matches_point[:, :, 2] + good_point = np.array([matches[index][array[0]] for index, array in + enumerate(np.argsort(np.abs(kp_src_angle - kp_sch_rotate_angle)))]) + + # 计算各点以first_good_point为原点的旋转角 + good_point_nan = (np.nan, np.nan) + good_point_pt = np.array([good_point_nan if dMatch is np.nan else (*kp_src[dMatch.trainIdx].pt, ) + for dMatch in good_point]) + good_point_origin_angle = get_points_origin_angle(good_point_pt[:, 0], good_point_pt[:, 1], + first_good_point_train) + threshold = round(5 / 360, 2) * 100 + point_bool = (np.abs(good_point_origin_angle - first_good_point_sch_origin_angle) / 360) * 100 < threshold + _, index = np.unique(good_point_pt[point_bool], return_index=True, axis=0) + good = good_point[point_bool] + good = good[index] + return good, int(first_good_point_angle), first_good_point + + def extract_good_points(self, kp_src, kp_sch, good, angle): + """ + 根据匹配点(good)数量,提取识别区域 + + Args: + kp_src: 关键点集 + kp_sch: 关键点集 + good: 描述符集 + angle: 旋转角度 + + Returns: + 范围,和置信度 + """ + len_good = len(good) + confidence, rect, target_img = 0, None, None + + if len_good == 1: + target_img, pypts, w_h_range = self._handle_one_good_point(kp_src=kp_src, kp_sch=kp_sch, good=good, + angle=angle) + elif len_good == 2: + target_img, pypts, w_h_range = self._handle_two_good_points(kp_src=kp_src, kp_sch=kp_sch, good=good) + elif len_good == 3: + target_img, pypts, w_h_range = self._handle_three_good_points(kp_src=kp_src, kp_sch=kp_sch, good=good) + else: + target_img, pypts, w_h_range = self._handle_many_good_points(kp_sch=kp_sch, kp_src=kp_src, good=good) + + if target_img is not None and target_img.any(): + confidence = self._cal_confidence(target_img) + # cv2.imshow('target', target_img) + # cv2.waitKey(0) + return pypts, w_h_range, confidence + + def _handle_one_good_point(self, kp_src, kp_sch, good, angle): + sch_point = get_keypoint_from_matches(kp=kp_sch, matches=good, mode='query') + src_point = get_keypoint_from_matches(kp=kp_src, matches=good, mode='train') + + scale = src_point[0].size / sch_point[0].size + h, w = self.im_search.shape[:-1] + _h, _w = h * scale, w * scale + src = np.float32(rectangle_transform(point=sch_point[0].pt, size=(h, w), mapping_point=src_point[0].pt, + mapping_size=(_h, _w), angle=angle)) + dst = np.float32([[0, 0], [w, 0], [0, h], [w, h]]) + output = self._perspective_transform(src=src, dst=dst) + pypts, w_h_range = self._get_perspective_area_rect(src=src) + return output, pypts, w_h_range def _handle_two_good_points(self, kp_sch, kp_src, good): """处理两对特征点的情况.""" @@ -168,7 +222,14 @@ def _handle_two_good_points(self, kp_sch, kp_src, good): pts_src1 = int(kp_src[good[0].trainIdx].pt[0]), int(kp_src[good[0].trainIdx].pt[1]) pts_src2 = int(kp_src[good[1].trainIdx].pt[0]), int(kp_src[good[1].trainIdx].pt[1]) - return self._get_origin_result_with_two_points(pts_sch1, pts_sch2, pts_src1, pts_src2) + result = self._get_origin_result_with_two_points(pts_sch1, pts_sch2, pts_src1, pts_src2) + if result: + middle_point, pypts, w_h_range = result + x_min, x_max, y_min, y_max, _, __ = w_h_range + target_img = self.im_source[y_min:y_max, x_min:x_max] + return target_img, pypts, w_h_range + else: + return None, None, None def _handle_three_good_points(self, kp_sch, kp_src, good): """处理三对特征点的情况.""" @@ -180,55 +241,15 @@ def _handle_three_good_points(self, kp_sch, kp_src, good): pts_src1 = int(kp_src[good[0].trainIdx].pt[0]), int(kp_src[good[0].trainIdx].pt[1]) pts_src2 = int((kp_src[good[1].trainIdx].pt[0] + kp_src[good[2].trainIdx].pt[0]) / 2), int( (kp_src[good[1].trainIdx].pt[1] + kp_src[good[2].trainIdx].pt[1]) / 2) - return self._get_origin_result_with_two_points(pts_sch1, pts_sch2, pts_src1, pts_src2) - - def _many_good_pts(self, kp_sch, kp_src, good): - """特征点匹配点对数目>=4个，可使用单矩阵映射,求出识别的目标区域.""" - sch_pts, img_pts = np.float32([kp_sch[m.queryIdx].pt for m in good]).reshape( - -1, 1, 2), np.float32([kp_src[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) - # M是转化矩阵 - M, mask = self._find_homography(sch_pts, img_pts) - matches_mask = mask.ravel().tolist() - # 从good中间筛选出更精确的点(假设good中大部分点为正确的，由ratio=0.7保障) - selected = [v for k, v in enumerate(good) if matches_mask[k]] - - # 针对所有的selected点再次计算出更精确的转化矩阵M来 - sch_pts, img_pts = np.float32([kp_sch[m.queryIdx].pt for m in selected]).reshape( - -1, 1, 2), np.float32([kp_src[m.trainIdx].pt for m in selected]).reshape(-1, 1, 2) - M, mask = self._find_homography(sch_pts, img_pts) - # 计算四个角矩阵变换后的坐标，也就是在大图中的目标区域的顶点坐标: - h, w = self.im_search.shape[:2] - h_s, w_s = self.im_source.shape[:2] - pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2) - dst = cv2.perspectiveTransform(pts, M) - - # trans numpy arrary to python list: [(a, b), (a1, b1), ...] - def cal_rect_pts(dst): - return [tuple(npt[0]) for npt in dst.astype(int).tolist()] - pypts = cal_rect_pts(dst) - # 注意：虽然4个角点有可能越出source图边界，但是(根据精确化映射单映射矩阵M线性机制)中点不会越出边界 - lt, br = pypts[0], pypts[2] - middle_point = int((lt[0] + br[0]) / 2), int((lt[1] + br[1]) / 2) - # 考虑到算出的目标矩阵有可能是翻转的情况，必须进行一次处理，确保映射后的“左上角”在图片中也是左上角点： - x_min, x_max = min(lt[0], br[0]), max(lt[0], br[0]) - y_min, y_max = min(lt[1], br[1]), max(lt[1], br[1]) - # 挑选出目标矩形区域可能会有越界情况，越界时直接将其置为边界： - # 超出左边界取0，超出右边界取w_s-1，超出下边界取0，超出上边界取h_s-1 - # 当x_min小于0时，取0。 x_max小于0时，取0。 - x_min, x_max = int(max(x_min, 0)), int(max(x_max, 0)) - # 当x_min大于w_s时，取值w_s-1。 x_max大于w_s-1时，取w_s-1。 - x_min, x_max = int(min(x_min, w_s - 1)), int(min(x_max, w_s - 1)) - # 当y_min小于0时，取0。 y_max小于0时，取0。 - y_min, y_max = int(max(y_min, 0)), int(max(y_max, 0)) - # 当y_min大于h_s时，取值h_s-1。 y_max大于h_s-1时，取h_s-1。 - y_min, y_max = int(min(y_min, h_s - 1)), int(min(y_max, h_s - 1)) - # 目标区域的角点，按左上、左下、右下、右上点序：(x_min,y_min)(x_min,y_max)(x_max,y_max)(x_max,y_min) - pts = np.float32([[x_min, y_min], [x_min, y_max], [ - x_max, y_max], [x_max, y_min]]).reshape(-1, 1, 2) - pypts = cal_rect_pts(pts) - - return middle_point, pypts, [x_min, x_max, y_min, y_max, w, h] + result = self._get_origin_result_with_two_points(pts_sch1, pts_sch2, pts_src1, pts_src2) + if result: + middle_point, pypts, w_h_range = result + x_min, x_max, y_min, y_max, _, __ = w_h_range + target_img = self.im_source[y_min:y_max, x_min:x_max] + return target_img, pypts, w_h_range + else: + return None, None, None def _get_origin_result_with_two_points(self, pts_sch1, pts_sch2, pts_src1, pts_src2): """返回两对有效匹配特征点情形下的识别结果.""" @@ -236,7 +257,8 @@ def _get_origin_result_with_two_points(self, pts_sch1, pts_sch2, pts_src1, pts_s middle_point = [int((pts_src1[0] + pts_src2[0]) / 2), int((pts_src1[1] + pts_src2[1]) / 2)] pypts = [] # 如果特征点同x轴或同y轴(无论src还是sch中)，均不能计算出目标矩形区域来，此时返回值同good=1情形 - if pts_sch1[0] == pts_sch2[0] or pts_sch1[1] == pts_sch2[1] or pts_src1[0] == pts_src2[0] or pts_src1[1] == pts_src2[1]: + if pts_sch1[0] == pts_sch2[0] or pts_sch1[1] == pts_sch2[1] or pts_src1[0] == pts_src2[0] or pts_src1[1] == \ + pts_src2[1]: return None # 计算x,y轴的缩放比例：x_scale、y_scale，从middle点扩张出目标区域:(注意整数计算要转成浮点数结果!) h, w = self.im_search.shape[:2] @@ -265,13 +287,149 @@ def _get_origin_result_with_two_points(self, pts_sch1, pts_sch2, pts_src1, pts_s return middle_point, pypts, [x_min, x_max, y_min, y_max, w, h] - def _find_homography(self, sch_pts, src_pts): + def _handle_many_good_points(self, kp_src, kp_sch, good): + """ + 特征点匹配数量>=4时,使用单矩阵映射,获取识别的目标图片 + + Args: + kp_sch: 关键点集 + kp_src: 关键点集 + good: 描述符集 + + Returns: + 透视变换后的图片 + """ + + sch_pts, img_pts = np.float32([kp_sch[m.queryIdx].pt for m in good]).reshape( + -1, 1, 2), np.float32([kp_src[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) + # M是转化矩阵 + M, mask = self._find_homography(sch_pts, img_pts) + # 计算四个角矩阵变换后的坐标，也就是在大图中的目标区域的顶点坐标: + h, w = self.im_search.shape[:-1] + pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2) + try: + dst: np.ndarray = cv2.perspectiveTransform(pts, M) + pypts = [tuple(npt[0]) for npt in dst.tolist()] + src = np.array([pypts[0], pypts[3], pypts[1], pypts[2]], dtype=np.float32) + dst = np.float32([[0, 0], [w, 0], [0, h], [w, h]]) + output = self._perspective_transform(src=src, dst=dst) + except cv2.error as err: + raise PerspectiveTransformError(err) + + pypts, w_h_range = self._get_perspective_area_rect(src=src) + return output, pypts, w_h_range + + def _get_perspective_area_rect(self, src): + """ + 根据矩形四个顶点坐标,获取在原图中的最大外接矩形 + + Args: + src: 目标图像中相应四边形顶点的坐标 + + Returns: + 最大外接矩形 + """ + h, w = self.im_source.shape[:-1] + + x = [int(i[0]) for i in src] + y = [int(i[1]) for i in src] + x_min, x_max = min(x), max(x) + y_min, y_max = min(y), max(y) + + def cal_rect_pts(dst): + return [tuple(npt[0]) for npt in dst.astype(int).tolist()] + + # 挑选出目标矩形区域可能会有越界情况，越界时直接将其置为边界： + # 超出左边界取0，超出右边界取w_s-1，超出下边界取0，超出上边界取h_s-1 + # 当x_min小于0时，取0。 x_max小于0时，取0。 + x_min, x_max = int(max(x_min, 0)), int(max(x_max, 0)) + # 当x_min大于w_s时，取值w_s-1。 x_max大于w_s-1时，取w_s-1。 + x_min, x_max = int(min(x_min, w - 1)), int(min(x_max, w - 1)) + # 当y_min小于0时，取0。 y_max小于0时，取0。 + y_min, y_max = int(max(y_min, 0)), int(max(y_max, 0)) + # 当y_min大于h_s时，取值h_s-1。 y_max大于h_s-1时，取h_s-1。 + y_min, y_max = int(min(y_min, h - 1)), int(min(y_max, h - 1)) + pts = np.float32([[x_min, y_min], [x_min, y_max], [ + x_max, y_max], [x_max, y_min]]).reshape(-1, 1, 2) + pypts = cal_rect_pts(pts) + return pypts, [x_min, x_max, y_min, y_max, w, h] + + def _perspective_transform(self, src, dst, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0): + """ + 根据四对对应点计算透视变换, 并裁剪相应图片 + + Args: + src: 目标图像中相应四边形顶点的坐标 (左上,右上,左下,右下) + dst: 源图像中四边形顶点的坐标 (左上,右上,左下,右下) + + Returns: + 透视变化后的图片 + """ + h, w = self.im_search.shape[:-1] + matrix = cv2.getPerspectiveTransform(src=src, dst=dst) + # warpPerspective https://github.com/opencv/opencv/issues/11784 + output = cv2.warpPerspective(self.im_source, matrix, (w, h), flags=flags, borderMode=borderMode, + borderValue=borderValue) + return output + + def show_match_image(self): + """Show how the keypoints matches.""" + from random import random + h_sch, w_sch = self.im_search.shape[:2] + h_src, w_src = self.im_source.shape[:2] + + # first you have to do the matching + self.find_best_result() + # then initialize the result image: + matching_info_img = np.zeros([max(h_sch, h_src), w_sch + w_src, 3], np.uint8) + matching_info_img[:h_sch, :w_sch, :] = self.im_search + matching_info_img[:h_src, w_sch:, :] = self.im_source + # render the match image at last: + for m in self.good: + color = tuple([int(random() * 255) for _ in range(3)]) + cv2.line(matching_info_img, (int(self.kp_sch[m.queryIdx].pt[0]), int(self.kp_sch[m.queryIdx].pt[1])), (int(self.kp_src[m.trainIdx].pt[0] + w_sch), int(self.kp_src[m.trainIdx].pt[1])), color) + + return matching_info_img + + def _cal_confidence(self, resize_img): + """计算confidence.""" + if self.rgb: + confidence = cal_rgb_confidence(resize_img, self.im_search) + else: + confidence = cal_ccoeff_confidence(resize_img, self.im_search) + # confidence修正 + confidence = (1 + confidence) / 2 + return confidence + + def init_detector(self): + """Init keypoint detector object.""" + self.detector = cv2.KAZE_create() + # create BFMatcher object: + self.matcher = cv2.BFMatcher(cv2.NORM_L1) # cv2.NORM_L1 cv2.NORM_L2 cv2.NORM_HAMMING(not useable) + + def get_keypoints_and_descriptors(self, image): + """获取图像特征点和描述符.""" + if image.shape[2] == 3: + image = cv2.cvtColor(image, code=cv2.COLOR_BGR2GRAY) + + keypoints, descriptors = self.detector.detectAndCompute(image, None) + if len(keypoints) < 2: + raise NoMatchPointError("Not enough feature points in input images !") + return keypoints, descriptors + + def match_keypoints(self, des_sch, des_src, k=2): + """Match descriptors (特征值匹配).""" + # 匹配两个图片中的特征点集，k=2表示每个特征点取出2个最匹配的对应点: + return self.matcher.knnMatch(des_sch, des_src, k=k) + + @staticmethod + def _find_homography(sch_pts, src_pts): """多组特征点对时，求取单向性矩阵.""" try: - M, mask = cv2.findHomography(sch_pts, src_pts, cv2.RANSAC, 5.0) - except Exception: - import traceback - traceback.print_exc() + M, mask = cv2.findHomography(sch_pts, src_pts, *CVRANSAC) + except cv2.error: + # import traceback + # traceback.print_exc() raise HomographyError("OpenCV error in _find_homography()...") else: if mask is None: @@ -288,4 +446,4 @@ def _target_error_check(self, w_h_range): raise MatchResultCheckError("In src_image, Taget area: width or height < 5 pixel.") # 如果矩形识别区域的宽和高，与sch_img的宽高差距超过5倍(屏幕像素差不可能有5倍)，认定为识别错误。 if tar_width < 0.2 * w or tar_width > 5 * w or tar_height < 0.2 * h or tar_height > 5 * h: - raise MatchResultCheckError("Target area is 5 times bigger or 0.2 times smaller than sch_img.") + raise MatchResultCheckError("Target area is 5 times bigger or 0.2 times smaller than sch_img.") \ No newline at end of file diff --git a/airtest/aircv/keypoint_matching_contrib.py b/airtest/aircv/keypoint_matching_contrib.py index 3fb42bf2..939a8076 100644 --- a/airtest/aircv/keypoint_matching_contrib.py +++ b/airtest/aircv/keypoint_matching_contrib.py @@ -54,11 +54,6 @@ def get_keypoints_and_descriptors(self, image): keypoints, descriptors = self.brief_extractor.compute(image, kp) return keypoints, descriptors - def match_keypoints(self, des_sch, des_src): - """Match descriptors (特征值匹配).""" - # 匹配两个图片中的特征点集，k=2表示每个特征点取出2个最匹配的对应点: - return self.matcher.knnMatch(des_sch, des_src, k=2) - class SIFTMatching(KeypointMatching): """SIFT Matching.""" @@ -92,11 +87,6 @@ def get_keypoints_and_descriptors(self, image): keypoints, descriptors = self.detector.detectAndCompute(image, None) return keypoints, descriptors - def match_keypoints(self, des_sch, des_src): - """Match descriptors (特征值匹配).""" - # 匹配两个图片中的特征点集，k=2表示每个特征点取出2个最匹配的对应点: - return self.matcher.knnMatch(des_sch, des_src, k=2) - class SURFMatching(KeypointMatching): """SURF Matching.""" @@ -131,7 +121,3 @@ def get_keypoints_and_descriptors(self, image): keypoints, descriptors = self.detector.detectAndCompute(image, None) return keypoints, descriptors - def match_keypoints(self, des_sch, des_src): - """Match descriptors (特征值匹配).""" - # 匹配两个图片中的特征点集，k=2表示每个特征点取出2个最匹配的对应点: - return self.matcher.knnMatch(des_sch, des_src, k=2) diff --git a/airtest/aircv/utils.py b/airtest/aircv/utils.py index 866de7c5..8721d62f 100644 --- a/airtest/aircv/utils.py +++ b/airtest/aircv/utils.py @@ -3,6 +3,7 @@ import cv2 import time +import math import numpy as np from PIL import Image @@ -105,3 +106,114 @@ def compress_image(pil_img, path, quality, max_size=None): if quality <= 0 or quality >= 100: raise Exception("SNAPSHOT_QUALITY (" + str(quality) + ") should be an integer in the range [1,99]") pil_img.save(path, quality=quality, optimize=True) + + +def get_keypoint_from_matches(kp, matches, mode): + res = [] + if mode == 'query': + for match in matches: + res.append(kp[match.queryIdx]) + elif mode == 'train': + for match in matches: + res.append(kp[match.trainIdx]) + + return res + + +def keypoint_distance(kp1, kp2): + """求两个keypoint的两点之间距离""" + if isinstance(kp1, cv2.KeyPoint): + kp1 = kp1.pt + elif isinstance(kp1, (list, tuple)): + kp1 = kp1 + else: + raise ValueError('kp1需要时keypoint或直接是坐标, kp1={}'.format(kp1)) + + if isinstance(kp2, cv2.KeyPoint): + kp2 = kp2.pt + elif isinstance(kp2, (list, tuple)): + kp2 = kp2 + else: + raise ValueError('kp2需要时keypoint或直接是坐标, kp1={}'.format(kp2)) + + x = kp1[0] - kp2[0] + y = kp1[1] - kp2[1] + return math.sqrt((x ** 2) + (y ** 2)) + + +def _mapping_angle_distance(distance, origin_angle, angle): + """ + + Args: + distance: 距离 + origin_angle: 对应原点的角度 + angle: 旋转角度 + + """ + _angle = origin_angle + angle + _y = distance * math.cos((math.pi * _angle) / 180) + _x = distance * math.sin((math.pi * _angle) / 180) + return round(_x, 3), round(_y, 3) + + +def rectangle_transform(point, size, mapping_point, mapping_size, angle): + """ + 根据point,找出mapping_point映射的矩形顶点坐标 + + Args: + point: 坐标在矩形中的坐标 + size: 矩形的大小(h, w) + mapping_point: 映射矩形的坐标 + mapping_size: 映射矩形的大小(h, w) + angle: 旋转角度 + + Returns: + + """ + h, w = size[0], size[1] + _h, _w = mapping_size[0], mapping_size[1] + + h_scale = _h / h + w_scale = _w / w + + tl = keypoint_distance((0, 0), point) # 左上 + tr = keypoint_distance((w, 0), point) # 右上 + bl = keypoint_distance((0, h), point) # 左下 + br = keypoint_distance((w, h), point) # 右下 + + # x = np.float32([point[1], point[1], (h - point[1]), (h - point[1])]) + # y = np.float32([point[0], (w - point[0]), point[0], (w - point[0])]) + # A, B, C, D = cv2.phase(x, y, angleInDegrees=True) + A = math.degrees(math.atan2(point[0], point[1])) + B = math.degrees(math.atan2((w - point[0]), point[1])) + C = math.degrees(math.atan2(point[0], (h - point[1]))) + D = math.degrees(math.atan2((w - point[0]), (h - point[1]))) + + new_tl = _mapping_angle_distance(tl, A, angle=angle) + new_tl = (-new_tl[0] * w_scale, -new_tl[1] * h_scale) + new_tl = (mapping_point[0] + new_tl[0], mapping_point[1] + new_tl[1]) + + new_tr = _mapping_angle_distance(tr, B, angle=angle) + new_tr = (new_tr[0] * w_scale, -new_tr[1] * h_scale) + new_tr = (mapping_point[0] + new_tr[0], mapping_point[1] + new_tr[1]) + + new_bl = _mapping_angle_distance(bl, C, angle=angle) + new_bl = (-new_bl[0] * w_scale, new_bl[1] * h_scale) + new_bl = (mapping_point[0] + new_bl[0], mapping_point[1] + new_bl[1]) + + new_br = _mapping_angle_distance(br, D, angle=angle) + new_br = (new_br[0] * w_scale, new_br[1] * h_scale) + new_br = (mapping_point[0] + new_br[0], mapping_point[1] + new_br[1]) + + return [new_tl, new_tr, new_bl, new_br] + + +def get_middle_point(w_h_range): + x, y = w_h_range[0], w_h_range[2] + width = w_h_range[1] - w_h_range[0] + height = w_h_range[3] - w_h_range[2] + middle_point = ( + x + width / 2, + y + height / 2 + ) + return middle_point