From 477402044c2a8abad57ca6a108aaa17d335937a9 Mon Sep 17 00:00:00 2001 From: qmchenry Date: Fri, 19 Jun 2020 10:13:03 -0400 Subject: [PATCH] Add average background color estimation [WIP] The intent is to mask the label out of the full screenshot of its rendered frame and compute the average of the remaining pixels. This is implemented but requires additional testing, in particular around partial-alpha pixels around the fringe of the label text. In many cases, this gives a comparable color as the existing estimate of averaging the four corner points. The inverse is also attempted, masking out the background of the screen capture and leaving the text pixels. This is not working yet, but left in to spur progress. It is not currently used in computations, only in report output. Other approaches for both of these should be considered. --- .../UILint/Checks/LabelContrastRatio.swift | 2 +- .../UILint/Checks/LabelUnexpectedFont.swift | 2 +- Sources/UILint/Element.swift | 20 ++-- Sources/UILint/Report.swift | 98 +++++++++++-------- Sources/UILint/ReportStyles.swift | 4 +- Sources/UILint/UILabelChecks.swift | 55 +++++++++-- Sources/UILint/UILint.swift | 6 +- Tests/UILintTests/UILabelTests.swift | 7 +- 8 files changed, 131 insertions(+), 63 deletions(-) diff --git a/Sources/UILint/Checks/LabelContrastRatio.swift b/Sources/UILint/Checks/LabelContrastRatio.swift index 770edd7..5cbe044 100644 --- a/Sources/UILint/Checks/LabelContrastRatio.swift +++ b/Sources/UILint/Checks/LabelContrastRatio.swift @@ -22,7 +22,7 @@ public struct LabelContrastRatio: Check { } public func findings(forElement element: Element, elements: [Element], context: LintingContext) -> [Finding] { - guard case let Element.label(font, _, _, textColor, base) = element else { return [] } + guard case let Element.label(font, _, _, textColor, _, _, base) = element else { return [] } guard let screenshot = context.screenshot, let cropped = crop(screenshot: screenshot, toWindowFrame: base.windowFrame), let bgColor = element.base.effectiveBackgroundColor, diff --git a/Sources/UILint/Checks/LabelUnexpectedFont.swift b/Sources/UILint/Checks/LabelUnexpectedFont.swift index ff6ccb3..82af1dd 100644 --- a/Sources/UILint/Checks/LabelUnexpectedFont.swift +++ b/Sources/UILint/Checks/LabelUnexpectedFont.swift @@ -11,7 +11,7 @@ public struct LabelUnexpectedFont: Check { public let description = "Label uses unexpected font." public func findings(forElement element: Element, elements: [Element], context: LintingContext) -> [Finding] { - guard case let Element.label(font, _, _, _, _) = element else { return [] } + guard case let Element.label(font, _, _, _, _, _, _) = element else { return [] } guard !UILintConfig.shared.expectedFontNames.isEmpty else { return [] } guard !UILintConfig.shared.expectedFontNames.contains(font.fontName) else { return [] } diff --git a/Sources/UILint/Element.swift b/Sources/UILint/Element.swift index 8b1b244..dce81a8 100644 --- a/Sources/UILint/Element.swift +++ b/Sources/UILint/Element.swift @@ -9,7 +9,8 @@ import UIKit public enum Element: Comparable, CustomDebugStringConvertible { - case label(font: UIFont, maxLines: Int, text: String, textColor: UIColor, base: Base) + case label(font: UIFont, maxLines: Int, text: String, textColor: UIColor, + measuredTextColor: UIColor?, measuredBackgroundColor: UIColor?, base: Base) case button(fontName: String?, fontSize: CGFloat?, title: String?, hasImage: Bool, base: Base) case image(image: UIImage?, imageAccessibilityLabel: String?, base: Base) case other(base: Base) @@ -27,13 +28,12 @@ public enum Element: Comparable, CustomDebugStringConvertible { public let contentMode: UIView.ContentMode public let accessibilityIdentifier: String? public let tag: Int - init(_ view: UIView, depth: Int, level: Int) { -// let screenshot = context.screenshot?.crop(to: view.windowFrame, viewSize: context.screenshot?.size) + init(_ view: UIView, depth: Int, level: Int, context: LintingContext) { + let screenshot = context.screenshot?.crop(to: view.windowFrame, viewSize: context.screenshot?.size) className = view.className windowFrame = view.windowFrame backgroundColor = view.backgroundColor -// effectiveBackgroundColor = screenshot?.effectiveBackgroundColor() - effectiveBackgroundColor = view.backgroundColor?.cgColor + effectiveBackgroundColor = screenshot?.effectiveBackgroundColor() let enabledGestureRecognizers = view.gestureRecognizers?.filter { $0.isEnabled }.count ?? 0 wantsTouches = (view is UIControl) || enabledGestureRecognizers > 0 consumesTouches = view.consumesTouches @@ -48,7 +48,7 @@ public enum Element: Comparable, CustomDebugStringConvertible { public var base: Base { switch self { - case .label(_, _, _, _, let base): return base + case .label(_, _, _, _, _, _, let base): return base case .button(_, _, _, _, let base): return base case .image(_, _, let base): return base case .other(let base): return base @@ -102,13 +102,17 @@ public enum Element: Comparable, CustomDebugStringConvertible { return descriptions.compactMap { $0 }.joined(separator: " ") } - init?(view: UIView, depth: Int, level: Int) { - let base = Base(view, depth: depth, level: level) + init?(view: UIView, depth: Int, level: Int, context: LintingContext) { + let base = Base(view, depth: depth, level: level, context: context) if let view = view as? UILabel { + let texture = context.screenshot?.crop(to: view.windowFrame, viewSize: context.screenshot!.size) + let extractor = LabelColorExtractor(screenshot: texture, label: view) self = Element.label(font: view.font, maxLines: view.numberOfLines, text: view.text ?? "", textColor: view.textColor, + measuredTextColor: extractor?.textColor, + measuredBackgroundColor: extractor?.backgroundColor, base: base) } else if let view = view as? UIButton { let font = view.titleLabel?.font diff --git a/Sources/UILint/Report.swift b/Sources/UILint/Report.swift index a3556ff..b7890a5 100644 --- a/Sources/UILint/Report.swift +++ b/Sources/UILint/Report.swift @@ -205,49 +205,65 @@ extension Report { } @discardableResult func draw(_ element: Element, draw performDraw: Bool = true) -> CGFloat { - var xPosition = padding switch element { - case .label(let font, _, let text, let textColor, let base): - let size0 = draw("\(element.base.depth) Label: \(font.pointSize)pt", attributes: body, - xPosition: xPosition, width: 140, updateHeight: false, draw: performDraw) - xPosition += 140 + padding - let size1 = draw("\(font.fontName)", attributes: body, xPosition: xPosition, width: 180, - updateHeight: false, draw: performDraw) - xPosition += 180 + padding - //let numberOfLines = element.numberOfLines(text: text, font: font, frame: base.windowFrame) - //let size2 = draw("\(numberOfLines) / \(maxLines) lines", attributes: body, xPosition: xPosition, - // width: 100, updateHeight: false, draw: performDraw) - //xPosition += 100 + padding - let colorSize = draw(textColor, xPosition: xPosition, updateHeight: false, draw: false) - xPosition = pageSize.width - 2 * colorSize.width - let size2 = draw(textColor, xPosition: xPosition, updateHeight: false, draw: performDraw) - xPosition += colorSize.width - let size3 = draw(base.effectiveBackgroundColor, xPosition: xPosition, updateHeight: false, - draw: performDraw) - let rowHeight = max(size0.height, size1.height, size2.height, size3.height) - if performDraw { - currentY += rowHeight + padding - } - let sizeText = draw(text, attributes: detail, xPosition: 40, draw: performDraw) - let height = rowHeight + padding + sizeText.height - return height - case .image(let image, let imageAccessibilityLabel, let base): - let size0 = draw("\(element.base.depth) \(base.className):", attributes: body, - xPosition: xPosition, width: 140, updateHeight: false, draw: performDraw) - xPosition += 140 + padding - let size1 = draw(imageAccessibilityLabel ?? "{no accessibility label}", attributes: body, - xPosition: xPosition, width: 240, updateHeight: false, draw: performDraw) - xPosition += 240 + padding - let size2 = draw(image, xPosition: xPosition, width: pageSize.width - xPosition - padding, - updateHeight: false, draw: performDraw) - let rowHeight = max(size0.height, size1.height, size2.height) + padding - if performDraw { - currentY += rowHeight - } - return rowHeight - default: break + case .label: return drawLabel(element, draw: performDraw) + case .image: return drawImage(element, draw: performDraw) + default: return 0 + } + } + + @discardableResult func drawLabel(_ element: Element, draw performDraw: Bool = true) -> CGFloat { + guard case let Element.label(font, maxLines, text, textColor, measuredTextColor, measuredBackgroundColor, base) + = element else { return 0 } + var xPosition = padding + let size0 = draw("\(element.base.depth) Label: \(font.pointSize)pt", attributes: body, + xPosition: xPosition, width: 140, updateHeight: false, draw: performDraw) + xPosition += 140 + padding + let size1 = draw("\(font.fontName)", attributes: body, xPosition: xPosition, width: 180, + updateHeight: false, draw: performDraw) + xPosition += 180 + padding + let numberOfLines = element.numberOfLines(text: text, font: font, frame: base.windowFrame) + let size2 = draw("\(numberOfLines) / \(maxLines) lines", attributes: body, xPosition: xPosition, + width: 100, updateHeight: false, draw: performDraw) + xPosition += 100 + padding + let bgColor = measuredBackgroundColor?.cgColor ?? base.effectiveBackgroundColor ?? UIColor.clear.cgColor + let contrast = textColor.cgColor.contrastRatio(with: bgColor) + draw("\(String(format: "%.2f", contrast ?? 0)):1", attributes: body, xPosition: xPosition, + width: 100, updateHeight: false, draw: performDraw) + xPosition = padding + if performDraw { currentY += max(size0.height, size1.height, size2.height) + padding } + let colorSize = draw(textColor, xPosition: xPosition, updateHeight: false, draw: false) + xPosition += draw(textColor, xPosition: xPosition, updateHeight: false, draw: performDraw).width + draw(measuredTextColor ?? .clear, xPosition: xPosition, updateHeight: false, draw: performDraw) + xPosition += colorSize.width + draw(base.backgroundColor ?? .clear, xPosition: xPosition, updateHeight: false, draw: performDraw) + xPosition += colorSize.width + draw(measuredBackgroundColor ?? .clear, xPosition: xPosition, updateHeight: false, draw: performDraw) + xPosition += colorSize.width + draw(base.effectiveBackgroundColor, xPosition: xPosition, updateHeight: false, draw: performDraw) + xPosition += colorSize.width + if performDraw { currentY += colorSize.height + padding } + let sizeText = draw(text, attributes: detail, xPosition: 40, draw: performDraw) + let height = colorSize.height + padding + sizeText.height + return height + } + + @discardableResult func drawImage(_ element: Element, draw performDraw: Bool = true) -> CGFloat { + guard case let Element.image(image, imageAccessibilityLabel, base) = element else { return 0 } + var xPosition = padding + let size0 = draw("\(element.base.depth) \(base.className):", attributes: body, + xPosition: xPosition, width: 140, updateHeight: false, draw: performDraw) + xPosition += 140 + padding + let size1 = draw(imageAccessibilityLabel ?? "{no accessibility label}", attributes: body, + xPosition: xPosition, width: 240, updateHeight: false, draw: performDraw) + xPosition += 240 + padding + let size2 = draw(image, xPosition: xPosition, width: pageSize.width - xPosition - padding, + updateHeight: false, draw: performDraw) + let rowHeight = max(size0.height, size1.height, size2.height) + padding + if performDraw { + currentY += rowHeight } - return 0 + return rowHeight } @discardableResult func draw(heirarchyElement element: Element, draw performDraw: Bool = true) -> CGFloat { diff --git a/Sources/UILint/ReportStyles.swift b/Sources/UILint/ReportStyles.swift index e0ed6ba..9b23bd2 100644 --- a/Sources/UILint/ReportStyles.swift +++ b/Sources/UILint/ReportStyles.swift @@ -8,7 +8,7 @@ import UIKit extension Report { - + var title1: [NSAttributedString.Key: Any] { let style = NSMutableParagraphStyle() style.alignment = .center @@ -86,6 +86,4 @@ extension Report { } return UIColor.yellow } - } - diff --git a/Sources/UILint/UILabelChecks.swift b/Sources/UILint/UILabelChecks.swift index 6a50574..9390c0f 100644 --- a/Sources/UILint/UILabelChecks.swift +++ b/Sources/UILint/UILabelChecks.swift @@ -10,12 +10,12 @@ import UIKit extension Element { var labelText: String? { - guard isLabel, case let Element.label(_, _, text, _, _) = self else { return nil } + guard isLabel, case let Element.label(_, _, text, _, _, _, _) = self else { return nil } return text } func isLabelTruncated() -> Bool { - guard isLabel, case let Element.label(font, maxLines, text, _, base) = self, + guard isLabel, case let Element.label(font, maxLines, text, _, _, _, base) = self, let frame = base.windowFrame else { return false } guard text.count > 0 else { return false } guard frame.width > 0 else { return true } @@ -23,7 +23,7 @@ extension Element { } func isLabelClippedVertically() -> Bool { - guard isLabel, case let Element.label(_, _, text, _, base) = self, + guard isLabel, case let Element.label(_, _, text, _, _, _, base) = self, let frame = base.windowFrame else { return false } guard text.count > 0 else { return false } guard frame.width > 0 else { return true } @@ -31,9 +31,9 @@ extension Element { } func isLabelOffscreen(windowSize: CGSize) -> Bool { - guard isLabel, case let Element.label(_, _, _, _, base) = self, + guard isLabel, case let Element.label(_, _, _, _, _, _, base) = self, let frame = base.windowFrame else { return false } - let windowRect = CGRect(origin: .zero, size: windowSize) + let windowRect = CGRect(origin: .zero, size: windowSize).rounded return windowRect.union(frame) != windowRect } @@ -44,7 +44,7 @@ extension Element { } func labelSize() -> CGSize { - guard isLabel, case let Element.label(font, _, text, _, base) = self, + guard isLabel, case let Element.label(font, _, text, _, _, _, base) = self, let frame = base.windowFrame else { return .zero } return (text as NSString).boundingRect(with: CGSize(width: frame.size.width, height: .greatestFiniteMagnitude), options: .usesLineFragmentOrigin, @@ -52,3 +52,46 @@ extension Element { context: nil).size } } + +class LabelColorExtractor { + var renderer: UIGraphicsImageRenderer! + var screenshot: UIImage + var label: UILabel { + didSet { + renderer = UIGraphicsImageRenderer(size: rect.size) + } + } + var rect: CGRect { + let scale = label.layer.contentsScale + let scaledRect = CGRect(x: 0, y: 0, width: label.frame.width/scale, height: label.frame.height/scale) + return scaledRect + } + + init?(screenshot: UIImage?, label: UILabel) { + guard let screenshot = screenshot else { return nil } + self.screenshot = screenshot + self.label = label + let scale = label.layer.contentsScale + let scaledSize = CGSize(width: label.frame.width/scale, height: label.frame.height/scale) + renderer = UIGraphicsImageRenderer(size: scaledSize) + } + + var backgroundColor: UIColor? { + let backgroundOnly = renderer.image { context in + screenshot.draw(in: rect) + context.cgContext.setBlendMode(.clear) + context.cgContext.setFontSize(label.font.pointSize) + label.layer.draw(in: context.cgContext) + } + let bgColor = backgroundOnly.averageColor(rect: rect) + return bgColor + } + + var textColor: UIColor? { + let textOnly = renderer.image { _ in + label.drawText(in: rect) + } + let textColor = textOnly.averageColor(rect: rect) + return textColor + } +} diff --git a/Sources/UILint/UILint.swift b/Sources/UILint/UILint.swift index 87abfc7..1d84bb8 100644 --- a/Sources/UILint/UILint.swift +++ b/Sources/UILint/UILint.swift @@ -20,7 +20,7 @@ public struct UILint { } let screenshot = grandparent.takeScreenshot() - context = LintingContext(windowSize: screenshot.size, + let context = LintingContext(windowSize: screenshot.size, screenshot: screenshot, safeAreaRect: grandparent.frame.inset(by: grandparent.safeAreaInsets), traitCollection: grandparentVC.traitCollection, @@ -29,11 +29,13 @@ public struct UILint { var currentDepth = 0 func recurse(_ view: UIView, level: Int) -> [Element] { - let viewOutput = [Element(view: view, depth: currentDepth, level: level)].compactMap { $0 } + let viewOutput = [Element(view: view, depth: currentDepth, level: level, context: context)] + .compactMap { $0 } currentDepth += 1 return view.allSubviews.compactMap { recurse($0, level: level + 1) }.reduce(viewOutput, +) } + self.context = context elements = recurse(grandparent, level: 0) } diff --git a/Tests/UILintTests/UILabelTests.swift b/Tests/UILintTests/UILabelTests.swift index f978943..9b4ece8 100644 --- a/Tests/UILintTests/UILabelTests.swift +++ b/Tests/UILintTests/UILabelTests.swift @@ -134,10 +134,15 @@ class UILabelTests: XCTestCase { let windowSize = CGSize(width: 320, height: 480) sut.view.frame = CGRect(origin: .zero, size: windowSize) + let context = LintingContext(windowSize: windowSize, + screenshot: nil, + safeAreaRect: CGRect(origin: .zero, size: windowSize), + traitCollection: UITraitCollection(), + shouldLint: nil) // need to find a way to sneak windowSize into QAElement.Base func isOffscreen(origin: CGPoint) -> Bool { let view = UILabel(frame: CGRect(origin: origin, size: size)) - let element = Element(view: view, depth: 0, level: 1)! + let element = Element(view: view, depth: 0, level: 1, context: context)! return element.isLabelOffscreen(windowSize: windowSize) } XCTAssertFalse(isOffscreen(origin: .zero))