Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

perf: Run getColors algorithm directly on CVPixelBuffer instead of costly UIImage conversion #9

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

perf: Run getColors algorithm directly on CVPixelBuffer instead of costly UIImage conversion #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
310 changes: 310 additions & 0 deletions ios/CVPixelBufferGetColors.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,310 @@
//
// CVPixelBufferGetColors.swift
// Colorwaver
//
// Created by Marc Rousavy on 22.09.21.
//


// Inspired by:
//
// UIImageColors.swift
// https://github.com/jathu/UIImageColors
//
// Created by Jathu Satkunarajah (@jathu) on 2015-06-11 - Toronto
//

import Foundation

import UIKit

public struct CVPixelBufferColors {
public var background: UIColor
public var primary: UIColor
public var secondary: UIColor
public var detail: UIColor

public init(background: UIColor, primary: UIColor, secondary: UIColor, detail: UIColor) {
self.background = background
self.primary = primary
self.secondary = secondary
self.detail = detail
}
}

public enum UIImageColorsQuality: CGFloat {
case lowest = 50 // 50px
case low = 100 // 100px
case high = 250 // 250px
case highest = 0 // No scale
}


/*
Extension on double that replicates UIColor methods. We DO NOT want these
exposed outside of the library because they don't make sense outside of the
context of UIImageColors.
*/
fileprivate extension Double {
private var r: Double {
return fmod(floor(self/1000000),1000000)
}

private var g: Double {
return fmod(floor(self/1000),1000)
}

private var b: Double {
return fmod(self,1000)
}

var isDarkColor: Bool {
return (r*0.2126) + (g*0.7152) + (b*0.0722) < 127.5
}

var isBlackOrWhite: Bool {
return (r > 232 && g > 232 && b > 232) || (r < 23 && g < 23 && b < 23)
}

func isDistinct(_ other: Double) -> Bool {
let _r = self.r
let _g = self.g
let _b = self.b
let o_r = other.r
let o_g = other.g
let o_b = other.b

return (fabs(_r-o_r) > 63.75 || fabs(_g-o_g) > 63.75 || fabs(_b-o_b) > 63.75)
&& !(fabs(_r-_g) < 7.65 && fabs(_r-_b) < 7.65 && fabs(o_r-o_g) < 7.65 && fabs(o_r-o_b) < 7.65)
}

func with(minSaturation: Double) -> Double {
// Ref: https://en.wikipedia.org/wiki/HSL_and_HSV

// Convert RGB to HSV

let _r = r/255
let _g = g/255
let _b = b/255
var H, S, V: Double
let M = fmax(_r,fmax(_g, _b))
var C = M-fmin(_r,fmin(_g, _b))

V = M
S = V == 0 ? 0:C/V

if minSaturation <= S {
return self
}

if C == 0 {
H = 0
} else if _r == M {
H = fmod((_g-_b)/C, 6)
} else if _g == M {
H = 2+((_b-_r)/C)
} else {
H = 4+((_r-_g)/C)
}

if H < 0 {
H += 6
}

// Back to RGB

C = V*minSaturation
let X = C*(1-fabs(fmod(H,2)-1))
var R, G, B: Double

switch H {
case 0...1:
R = C
G = X
B = 0
case 1...2:
R = X
G = C
B = 0
case 2...3:
R = 0
G = C
B = X
case 3...4:
R = 0
G = X
B = C
case 4...5:
R = X
G = 0
B = C
case 5..<6:
R = C
G = 0
B = X
default:
R = 0
G = 0
B = 0
}

let m = V-C

return (floor((R + m)*255)*1000000)+(floor((G + m)*255)*1000)+floor((B + m)*255)
}

func isContrasting(_ color: Double) -> Bool {
let bgLum = (0.2126*r)+(0.7152*g)+(0.0722*b)+12.75
let fgLum = (0.2126*color.r)+(0.7152*color.g)+(0.0722*color.b)+12.75
if bgLum > fgLum {
return 1.6 < bgLum/fgLum
} else {
return 1.6 < fgLum/bgLum
}
}

var uicolor: UIColor {
return UIColor(red: CGFloat(r)/255, green: CGFloat(g)/255, blue: CGFloat(b)/255, alpha: 1)
}

var pretty: String {
return "\(Int(self.r)), \(Int(self.g)), \(Int(self.b))"
}
}

extension CVPixelBuffer {

private func resizeForUIImageColors(newSize: CGSize) -> CVPixelBuffer? {
UIGraphicsBeginImageContextWithOptions(newSize, false, 0)
defer {
UIGraphicsEndImageContext()
}
//self.draw(in: CGRect(x: 0, y: 0, width: newSize.width, height: newSize.height))
return nil
}

public func getColors(quality: UIImageColorsQuality = .high) -> CVPixelBufferColors? {
let width = CVPixelBufferGetWidth(self)
let height = CVPixelBufferGetHeight(self)

let scaleDownSize: CGSize
if quality == .highest {
scaleDownSize = CGSize(width: width, height: height)
} else {
if width < height {
let ratio = CGFloat(height) / CGFloat(width)
scaleDownSize = CGSize(width: quality.rawValue / ratio, height: quality.rawValue)
} else {
let ratio = CGFloat(width) / CGFloat(height)
scaleDownSize = CGSize(width: quality.rawValue, height: quality.rawValue / ratio)
}
}

guard let resizedImage = self.resizeForUIImageColors(newSize: scaleDownSize) else { return nil }

let threshold = Int(Double(height) * 0.01)
var proposed: [Double] = [-1,-1,-1,-1]

let imageColors = NSCountedSet(capacity: width * height)
for x in 0 ..< width {
for y in 0 ..< height {
let pixel: Int = ((width * y) + x) * 4
if data[pixel+3] > 127 {
imageColors.add((Double(data[pixel+2])*1000000)+(Double(data[pixel+1])*1000)+(Double(data[pixel])))
}
}
}

let sortedColorComparator: Comparator = { (main, other) -> ComparisonResult in
let m = main as! UIImageColorsCounter, o = other as! UIImageColorsCounter
if m.count < o.count {
return .orderedDescending
} else if m.count == o.count {
return .orderedSame
} else {
return .orderedAscending
}
}

var enumerator = imageColors.objectEnumerator()
var sortedColors = NSMutableArray(capacity: imageColors.count)
while let K = enumerator.nextObject() as? Double {
let C = imageColors.count(for: K)
if threshold < C {
sortedColors.add(UIImageColorsCounter(color: K, count: C))
}
}
sortedColors.sort(comparator: sortedColorComparator)

var proposedEdgeColor: UIImageColorsCounter
if 0 < sortedColors.count {
proposedEdgeColor = sortedColors.object(at: 0) as! UIImageColorsCounter
} else {
proposedEdgeColor = UIImageColorsCounter(color: 0, count: 1)
}

if proposedEdgeColor.color.isBlackOrWhite && 0 < sortedColors.count {
for i in 1..<sortedColors.count {
let nextProposedEdgeColor = sortedColors.object(at: i) as! UIImageColorsCounter
if Double(nextProposedEdgeColor.count)/Double(proposedEdgeColor.count) > 0.3 {
if !nextProposedEdgeColor.color.isBlackOrWhite {
proposedEdgeColor = nextProposedEdgeColor
break
}
} else {
break
}
}
}
proposed[0] = proposedEdgeColor.color

enumerator = imageColors.objectEnumerator()
sortedColors.removeAllObjects()
sortedColors = NSMutableArray(capacity: imageColors.count)
let findDarkTextColor = !proposed[0].isDarkColor

while var K = enumerator.nextObject() as? Double {
K = K.with(minSaturation: 0.15)
if K.isDarkColor == findDarkTextColor {
let C = imageColors.count(for: K)
sortedColors.add(UIImageColorsCounter(color: K, count: C))
}
}
sortedColors.sort(comparator: sortedColorComparator)

for color in sortedColors {
let color = (color as! UIImageColorsCounter).color

if proposed[1] == -1 {
if color.isContrasting(proposed[0]) {
proposed[1] = color
}
} else if proposed[2] == -1 {
if !color.isContrasting(proposed[0]) || !proposed[1].isDistinct(color) {
continue
}
proposed[2] = color
} else if proposed[3] == -1 {
if !color.isContrasting(proposed[0]) || !proposed[2].isDistinct(color) || !proposed[1].isDistinct(color) {
continue
}
proposed[3] = color
break
}
}

let isDarkBackground = proposed[0].isDarkColor
for i in 1...3 {
if proposed[i] == -1 {
proposed[i] = isDarkBackground ? 255255255:0
}
}

return UIImageColors(
background: proposed[0].uicolor,
primary: proposed[1].uicolor,
secondary: proposed[2].uicolor,
detail: proposed[3].uicolor
)
}
}
12 changes: 8 additions & 4 deletions ios/Colorwaver.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
B869228626DA27AC00186D7D /* PaletteFrameProcessorPlugin.m in Sources */ = {isa = PBXBuildFile; fileRef = B869228526DA27AC00186D7D /* PaletteFrameProcessorPlugin.m */; };
B869228826DA29AA00186D7D /* PaletteFrameProcessorPlugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = B869228726DA29AA00186D7D /* PaletteFrameProcessorPlugin.swift */; };
B89B4A6D26DA48ED0063A3DB /* UIColor+hexString.swift in Sources */ = {isa = PBXBuildFile; fileRef = B89B4A6C26DA48ED0063A3DB /* UIColor+hexString.swift */; };
B8D9432D26FB77320087D73C /* CVPixelBufferGetColors.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8D9432C26FB77320087D73C /* CVPixelBufferGetColors.swift */; };
/* End PBXBuildFile section */

/* Begin PBXContainerItemProxy section */
Expand Down Expand Up @@ -51,6 +52,7 @@
B869228526DA27AC00186D7D /* PaletteFrameProcessorPlugin.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = PaletteFrameProcessorPlugin.m; sourceTree = "<group>"; };
B869228726DA29AA00186D7D /* PaletteFrameProcessorPlugin.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PaletteFrameProcessorPlugin.swift; sourceTree = "<group>"; };
B89B4A6C26DA48ED0063A3DB /* UIColor+hexString.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "UIColor+hexString.swift"; sourceTree = "<group>"; };
B8D9432C26FB77320087D73C /* CVPixelBufferGetColors.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CVPixelBufferGetColors.swift; sourceTree = "<group>"; };
DDF131CE188414F6F7234528 /* Pods-Colorwaver-ColorwaverTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Colorwaver-ColorwaverTests.release.xcconfig"; path = "Target Support Files/Pods-Colorwaver-ColorwaverTests/Pods-Colorwaver-ColorwaverTests.release.xcconfig"; sourceTree = "<group>"; };
ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; };
/* End PBXFileReference section */
Expand Down Expand Up @@ -117,6 +119,7 @@
B869228526DA27AC00186D7D /* PaletteFrameProcessorPlugin.m */,
B869228726DA29AA00186D7D /* PaletteFrameProcessorPlugin.swift */,
B89B4A6C26DA48ED0063A3DB /* UIColor+hexString.swift */,
B8D9432C26FB77320087D73C /* CVPixelBufferGetColors.swift */,
);
name = Colorwaver;
sourceTree = "<group>";
Expand Down Expand Up @@ -221,9 +224,7 @@
TestTargetID = 13B07F861A680F5B00A75B9A;
};
13B07F861A680F5B00A75B9A = {
DevelopmentTeam = CJW62Q77E7;
LastSwiftMigration = 1250;
ProvisioningStyle = Automatic;
};
};
};
Expand Down Expand Up @@ -395,6 +396,7 @@
B89B4A6D26DA48ED0063A3DB /* UIColor+hexString.swift in Sources */,
13B07FBC1A68108700A75B9A /* AppDelegate.m in Sources */,
B869228826DA29AA00186D7D /* PaletteFrameProcessorPlugin.swift in Sources */,
B8D9432D26FB77320087D73C /* CVPixelBufferGetColors.swift in Sources */,
B869228626DA27AC00186D7D /* PaletteFrameProcessorPlugin.m in Sources */,
B869228426DA1F9100186D7D /* File.swift in Sources */,
13B07FC11A68108700A75B9A /* main.m in Sources */,
Expand Down Expand Up @@ -475,6 +477,7 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = CJW62Q77E7;
Expand All @@ -491,7 +494,7 @@
);
PRODUCT_BUNDLE_IDENTIFIER = com.mrousavy.colorwaver;
PRODUCT_NAME = Colorwaver;
PROVISIONING_PROFILE_SPECIFIER = "match Development com.mrousavy.colorwaver";
PROVISIONING_PROFILE_SPECIFIER = "";
SWIFT_OBJC_BRIDGING_HEADER = "Colorwaver-Bridging-Header.h";
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
SWIFT_VERSION = 5.2;
Expand All @@ -505,6 +508,7 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = CJW62Q77E7;
Expand All @@ -520,7 +524,7 @@
);
PRODUCT_BUNDLE_IDENTIFIER = com.mrousavy.colorwaver;
PRODUCT_NAME = Colorwaver;
PROVISIONING_PROFILE_SPECIFIER = "match Development com.mrousavy.colorwaver";
PROVISIONING_PROFILE_SPECIFIER = "";
SWIFT_OBJC_BRIDGING_HEADER = "Colorwaver-Bridging-Header.h";
SWIFT_VERSION = 5.2;
VERSIONING_SYSTEM = "apple-generic";
Expand Down