Skip to content

Commit

Permalink
Add average background color estimation [WIP]
Browse files Browse the repository at this point in the history 
The intent is to mask the label out of the full screenshot of
its rendered frame and compute the average of the remaining
pixels. This is implemented but requires additional testing, in
particular around partial-alpha pixels around the fringe of the
label text. In many cases, this gives a comparable color as the
existing estimate of averaging the four corner points.

The inverse is also attempted, masking out the
background of the screen capture and leaving the text pixels.
This is not working yet, but left in to spur progress. It is not
currently used in computations, only in report output.

Other approaches for both of these should be considered.
  • Loading branch information
@qmchenry
qmchenry committed Jun 19, 2020
1 parent 7fab1db commit 4774020
Show file tree
Hide file tree
Showing 8 changed files with 131 additions and 63 deletions.
2 changes: 1 addition & 1 deletion Sources/UILint/Checks/LabelContrastRatio.swift
View file
Expand Up @@ -22,7 +22,7 @@ public struct LabelContrastRatio: Check {
}

public func findings(forElement element: Element, elements: [Element], context: LintingContext) -> [Finding] {
guard case let Element.label(font, _, _, textColor, base) = element else { return [] }
guard case let Element.label(font, _, _, textColor, _, _, base) = element else { return [] }
guard let screenshot = context.screenshot,
let cropped = crop(screenshot: screenshot, toWindowFrame: base.windowFrame),
let bgColor = element.base.effectiveBackgroundColor,
Expand Down
2 changes: 1 addition & 1 deletion Sources/UILint/Checks/LabelUnexpectedFont.swift
View file
Expand Up @@ -11,7 +11,7 @@ public struct LabelUnexpectedFont: Check {
public let description = "Label uses unexpected font."

public func findings(forElement element: Element, elements: [Element], context: LintingContext) -> [Finding] {
guard case let Element.label(font, _, _, _, _) = element else { return [] }
guard case let Element.label(font, _, _, _, _, _, _) = element else { return [] }
guard !UILintConfig.shared.expectedFontNames.isEmpty else { return [] }
guard !UILintConfig.shared.expectedFontNames.contains(font.fontName) else { return [] }

Expand Down
20 changes: 12 additions & 8 deletions Sources/UILint/Element.swift
View file
Expand Up @@ -9,7 +9,8 @@ import UIKit

public enum Element: Comparable, CustomDebugStringConvertible {

case label(font: UIFont, maxLines: Int, text: String, textColor: UIColor, base: Base)
case label(font: UIFont, maxLines: Int, text: String, textColor: UIColor,
measuredTextColor: UIColor?, measuredBackgroundColor: UIColor?, base: Base)
case button(fontName: String?, fontSize: CGFloat?, title: String?, hasImage: Bool, base: Base)
case image(image: UIImage?, imageAccessibilityLabel: String?, base: Base)
case other(base: Base)
Expand All @@ -27,13 +28,12 @@ public enum Element: Comparable, CustomDebugStringConvertible {
public let contentMode: UIView.ContentMode
public let accessibilityIdentifier: String?
public let tag: Int
init(_ view: UIView, depth: Int, level: Int) {
// let screenshot = context.screenshot?.crop(to: view.windowFrame, viewSize: context.screenshot?.size)
init(_ view: UIView, depth: Int, level: Int, context: LintingContext) {
let screenshot = context.screenshot?.crop(to: view.windowFrame, viewSize: context.screenshot?.size)
className = view.className
windowFrame = view.windowFrame
backgroundColor = view.backgroundColor
// effectiveBackgroundColor = screenshot?.effectiveBackgroundColor()
effectiveBackgroundColor = view.backgroundColor?.cgColor
effectiveBackgroundColor = screenshot?.effectiveBackgroundColor()
let enabledGestureRecognizers = view.gestureRecognizers?.filter { $0.isEnabled }.count ?? 0
wantsTouches = (view is UIControl) || enabledGestureRecognizers > 0
consumesTouches = view.consumesTouches
Expand All @@ -48,7 +48,7 @@ public enum Element: Comparable, CustomDebugStringConvertible {

public var base: Base {
switch self {
case .label(_, _, _, _, let base): return base
case .label(_, _, _, _, _, _, let base): return base
case .button(_, _, _, _, let base): return base
case .image(_, _, let base): return base
case .other(let base): return base
Expand Down Expand Up @@ -102,13 +102,17 @@ public enum Element: Comparable, CustomDebugStringConvertible {
return descriptions.compactMap { $0 }.joined(separator: " ")
}

init?(view: UIView, depth: Int, level: Int) {
let base = Base(view, depth: depth, level: level)
init?(view: UIView, depth: Int, level: Int, context: LintingContext) {
let base = Base(view, depth: depth, level: level, context: context)
if let view = view as? UILabel {
let texture = context.screenshot?.crop(to: view.windowFrame, viewSize: context.screenshot!.size)
let extractor = LabelColorExtractor(screenshot: texture, label: view)
self = Element.label(font: view.font,
maxLines: view.numberOfLines,
text: view.text ?? "",
textColor: view.textColor,
measuredTextColor: extractor?.textColor,
measuredBackgroundColor: extractor?.backgroundColor,
base: base)
} else if let view = view as? UIButton {
let font = view.titleLabel?.font
Expand Down
98 changes: 57 additions & 41 deletions Sources/UILint/Report.swift
View file
Expand Up @@ -205,49 +205,65 @@ extension Report {
}

@discardableResult func draw(_ element: Element, draw performDraw: Bool = true) -> CGFloat {
var xPosition = padding
switch element {
case .label(let font, _, let text, let textColor, let base):
let size0 = draw("\(element.base.depth) Label: \(font.pointSize)pt", attributes: body,
xPosition: xPosition, width: 140, updateHeight: false, draw: performDraw)
xPosition += 140 + padding
let size1 = draw("\(font.fontName)", attributes: body, xPosition: xPosition, width: 180,
updateHeight: false, draw: performDraw)
xPosition += 180 + padding
//let numberOfLines = element.numberOfLines(text: text, font: font, frame: base.windowFrame)
//let size2 = draw("\(numberOfLines) / \(maxLines) lines", attributes: body, xPosition: xPosition,
// width: 100, updateHeight: false, draw: performDraw)
//xPosition += 100 + padding
let colorSize = draw(textColor, xPosition: xPosition, updateHeight: false, draw: false)
xPosition = pageSize.width - 2 * colorSize.width
let size2 = draw(textColor, xPosition: xPosition, updateHeight: false, draw: performDraw)
xPosition += colorSize.width
let size3 = draw(base.effectiveBackgroundColor, xPosition: xPosition, updateHeight: false,
draw: performDraw)
let rowHeight = max(size0.height, size1.height, size2.height, size3.height)
if performDraw {
currentY += rowHeight + padding
}
let sizeText = draw(text, attributes: detail, xPosition: 40, draw: performDraw)
let height = rowHeight + padding + sizeText.height
return height
case .image(let image, let imageAccessibilityLabel, let base):
let size0 = draw("\(element.base.depth) \(base.className):", attributes: body,
xPosition: xPosition, width: 140, updateHeight: false, draw: performDraw)
xPosition += 140 + padding
let size1 = draw(imageAccessibilityLabel ?? "{no accessibility label}", attributes: body,
xPosition: xPosition, width: 240, updateHeight: false, draw: performDraw)
xPosition += 240 + padding
let size2 = draw(image, xPosition: xPosition, width: pageSize.width - xPosition - padding,
updateHeight: false, draw: performDraw)
let rowHeight = max(size0.height, size1.height, size2.height) + padding
if performDraw {
currentY += rowHeight
}
return rowHeight
default: break
case .label: return drawLabel(element, draw: performDraw)
case .image: return drawImage(element, draw: performDraw)
default: return 0
}
}

@discardableResult func drawLabel(_ element: Element, draw performDraw: Bool = true) -> CGFloat {
guard case let Element.label(font, maxLines, text, textColor, measuredTextColor, measuredBackgroundColor, base)
= element else { return 0 }
var xPosition = padding
let size0 = draw("\(element.base.depth) Label: \(font.pointSize)pt", attributes: body,
xPosition: xPosition, width: 140, updateHeight: false, draw: performDraw)
xPosition += 140 + padding
let size1 = draw("\(font.fontName)", attributes: body, xPosition: xPosition, width: 180,
updateHeight: false, draw: performDraw)
xPosition += 180 + padding
let numberOfLines = element.numberOfLines(text: text, font: font, frame: base.windowFrame)
let size2 = draw("\(numberOfLines) / \(maxLines) lines", attributes: body, xPosition: xPosition,
width: 100, updateHeight: false, draw: performDraw)
xPosition += 100 + padding
let bgColor = measuredBackgroundColor?.cgColor ?? base.effectiveBackgroundColor ?? UIColor.clear.cgColor
let contrast = textColor.cgColor.contrastRatio(with: bgColor)
draw("\(String(format: "%.2f", contrast ?? 0)):1", attributes: body, xPosition: xPosition,
width: 100, updateHeight: false, draw: performDraw)
xPosition = padding
if performDraw { currentY += max(size0.height, size1.height, size2.height) + padding }
let colorSize = draw(textColor, xPosition: xPosition, updateHeight: false, draw: false)
xPosition += draw(textColor, xPosition: xPosition, updateHeight: false, draw: performDraw).width
draw(measuredTextColor ?? .clear, xPosition: xPosition, updateHeight: false, draw: performDraw)
xPosition += colorSize.width
draw(base.backgroundColor ?? .clear, xPosition: xPosition, updateHeight: false, draw: performDraw)
xPosition += colorSize.width
draw(measuredBackgroundColor ?? .clear, xPosition: xPosition, updateHeight: false, draw: performDraw)
xPosition += colorSize.width
draw(base.effectiveBackgroundColor, xPosition: xPosition, updateHeight: false, draw: performDraw)
xPosition += colorSize.width
if performDraw { currentY += colorSize.height + padding }
let sizeText = draw(text, attributes: detail, xPosition: 40, draw: performDraw)
let height = colorSize.height + padding + sizeText.height
return height
}

@discardableResult func drawImage(_ element: Element, draw performDraw: Bool = true) -> CGFloat {
guard case let Element.image(image, imageAccessibilityLabel, base) = element else { return 0 }
var xPosition = padding
let size0 = draw("\(element.base.depth) \(base.className):", attributes: body,
xPosition: xPosition, width: 140, updateHeight: false, draw: performDraw)
xPosition += 140 + padding
let size1 = draw(imageAccessibilityLabel ?? "{no accessibility label}", attributes: body,
xPosition: xPosition, width: 240, updateHeight: false, draw: performDraw)
xPosition += 240 + padding
let size2 = draw(image, xPosition: xPosition, width: pageSize.width - xPosition - padding,
updateHeight: false, draw: performDraw)
let rowHeight = max(size0.height, size1.height, size2.height) + padding
if performDraw {
currentY += rowHeight
}
return 0
return rowHeight
}

@discardableResult func draw(heirarchyElement element: Element, draw performDraw: Bool = true) -> CGFloat {
Expand Down
4 changes: 1 addition & 3 deletions Sources/UILint/ReportStyles.swift
View file
Expand Up @@ -8,7 +8,7 @@
import UIKit

extension Report {

var title1: [NSAttributedString.Key: Any] {
let style = NSMutableParagraphStyle()
style.alignment = .center
Expand Down Expand Up @@ -86,6 +86,4 @@ extension Report {
}
return UIColor.yellow
}

}

55 changes: 49 additions & 6 deletions Sources/UILint/UILabelChecks.swift
View file
Expand Up @@ -10,30 +10,30 @@ import UIKit
extension Element {

var labelText: String? {
guard isLabel, case let Element.label(_, _, text, _, _) = self else { return nil }
guard isLabel, case let Element.label(_, _, text, _, _, _, _) = self else { return nil }
return text
}

func isLabelTruncated() -> Bool {
guard isLabel, case let Element.label(font, maxLines, text, _, base) = self,
guard isLabel, case let Element.label(font, maxLines, text, _, _, _, base) = self,
let frame = base.windowFrame else { return false }
guard text.count > 0 else { return false }
guard frame.width > 0 else { return true }
return maxLines > 0 ? numberOfLines(text: text, font: font, frame: frame) > maxLines : false
}

func isLabelClippedVertically() -> Bool {
guard isLabel, case let Element.label(_, _, text, _, base) = self,
guard isLabel, case let Element.label(_, _, text, _, _, _, base) = self,
let frame = base.windowFrame else { return false }
guard text.count > 0 else { return false }
guard frame.width > 0 else { return true }
return labelSize().height.rounded() > frame.size.height.rounded()
}

func isLabelOffscreen(windowSize: CGSize) -> Bool {
guard isLabel, case let Element.label(_, _, _, _, base) = self,
guard isLabel, case let Element.label(_, _, _, _, _, _, base) = self,
let frame = base.windowFrame else { return false }
let windowRect = CGRect(origin: .zero, size: windowSize)
let windowRect = CGRect(origin: .zero, size: windowSize).rounded
return windowRect.union(frame) != windowRect
}

Expand All @@ -44,11 +44,54 @@ extension Element {
}

func labelSize() -> CGSize {
guard isLabel, case let Element.label(font, _, text, _, base) = self,
guard isLabel, case let Element.label(font, _, text, _, _, _, base) = self,
let frame = base.windowFrame else { return .zero }
return (text as NSString).boundingRect(with: CGSize(width: frame.size.width, height: .greatestFiniteMagnitude),
options: .usesLineFragmentOrigin,
attributes: [.font: font],
context: nil).size
}
}

class LabelColorExtractor {
var renderer: UIGraphicsImageRenderer!
var screenshot: UIImage
var label: UILabel {
didSet {
renderer = UIGraphicsImageRenderer(size: rect.size)
}
}
var rect: CGRect {
let scale = label.layer.contentsScale
let scaledRect = CGRect(x: 0, y: 0, width: label.frame.width/scale, height: label.frame.height/scale)
return scaledRect
}

init?(screenshot: UIImage?, label: UILabel) {
guard let screenshot = screenshot else { return nil }
self.screenshot = screenshot
self.label = label
let scale = label.layer.contentsScale
let scaledSize = CGSize(width: label.frame.width/scale, height: label.frame.height/scale)
renderer = UIGraphicsImageRenderer(size: scaledSize)
}

var backgroundColor: UIColor? {
let backgroundOnly = renderer.image { context in
screenshot.draw(in: rect)
context.cgContext.setBlendMode(.clear)
context.cgContext.setFontSize(label.font.pointSize)
label.layer.draw(in: context.cgContext)
}
let bgColor = backgroundOnly.averageColor(rect: rect)
return bgColor
}

var textColor: UIColor? {
let textOnly = renderer.image { _ in
label.drawText(in: rect)
}
let textColor = textOnly.averageColor(rect: rect)
return textColor
}
}
6 changes: 4 additions & 2 deletions Sources/UILint/UILint.swift
View file
Expand Up @@ -20,7 +20,7 @@ public struct UILint {
}

let screenshot = grandparent.takeScreenshot()
context = LintingContext(windowSize: screenshot.size,
let context = LintingContext(windowSize: screenshot.size,
screenshot: screenshot,
safeAreaRect: grandparent.frame.inset(by: grandparent.safeAreaInsets),
traitCollection: grandparentVC.traitCollection,
Expand All @@ -29,11 +29,13 @@ public struct UILint {
var currentDepth = 0

func recurse(_ view: UIView, level: Int) -> [Element] {
let viewOutput = [Element(view: view, depth: currentDepth, level: level)].compactMap { $0 }
let viewOutput = [Element(view: view, depth: currentDepth, level: level, context: context)]
.compactMap { $0 }
currentDepth += 1
return view.allSubviews.compactMap { recurse($0, level: level + 1) }.reduce(viewOutput, +)
}

self.context = context
elements = recurse(grandparent, level: 0)
}

Expand Down
7 changes: 6 additions & 1 deletion Tests/UILintTests/UILabelTests.swift
View file
Expand Up @@ -134,10 +134,15 @@ class UILabelTests: XCTestCase {
let windowSize = CGSize(width: 320, height: 480)
sut.view.frame = CGRect(origin: .zero, size: windowSize)

let context = LintingContext(windowSize: windowSize,
screenshot: nil,
safeAreaRect: CGRect(origin: .zero, size: windowSize),
traitCollection: UITraitCollection(),
shouldLint: nil)
// need to find a way to sneak windowSize into QAElement.Base
func isOffscreen(origin: CGPoint) -> Bool {
let view = UILabel(frame: CGRect(origin: origin, size: size))
let element = Element(view: view, depth: 0, level: 1)!
let element = Element(view: view, depth: 0, level: 1, context: context)!
return element.isLabelOffscreen(windowSize: windowSize)
}
XCTAssertFalse(isOffscreen(origin: .zero))
Expand Down

0 comments on commit 4774020

Please sign in to comment.