Skip to content

Commit

Permalink
feat(predictions): add implementation for no light support (#3618)
Browse files Browse the repository at this point in the history 
* feat(predictions): add implementation for no light support

* fix swiftlint issue

* address review comments

* address review comments

* address review comments
  • Loading branch information
@thisisabhash
thisisabhash committed Apr 17, 2024
1 parent 74f3788 commit 0148783
Show file tree
Hide file tree
Showing 11 changed files with 252 additions and 72 deletions.
4 changes: 3 additions & 1 deletion AmplifyPlugins/Predictions/AWSPredictionsPlugin/Liveness/Events/LivenessEvent.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ public enum LivenessEventKind {
self.rawValue = rawValue
}

public static let challenge = Self(rawValue: "ServerSessionInformationEvent")
public static let sessionInformation = Self(rawValue: "ServerSessionInformationEvent")
public static let disconnect = Self(rawValue: "DisconnectionEvent")
public static let challenge = Self(rawValue: "ChallengeEvent")
}
case server(Server)

Expand Down Expand Up @@ -59,6 +60,7 @@ extension LivenessEventKind: CustomDebugStringConvertible {
public var debugDescription: String {
switch self {
case .server(.challenge): return ".server(.challenge)"
case .server(.sessionInformation): return ".server(.sessionInformation)"
case .server(.disconnect): return ".server(.disconnect)"
case .client(.initialFaceDetected): return ".client(.initialFaceDetected)"
case .client(.video): return ".client(.video)"
Expand Down
33 changes: 27 additions & 6 deletions ...ness/Events/LivenessFinalCientEvent.swift → ...ess/Events/LivenessFinalClientEvent.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ public struct FinalClientEvent {

extension LivenessEvent where T == FinalClientEvent {
@_spi(PredictionsFaceLiveness)
public static func final(event: FinalClientEvent) throws -> Self {

let clientEvent = ClientSessionInformationEvent(
challenge: .init(
faceMovementAndLightChallenge: .init(
public static func final(event: FinalClientEvent,
challenge: Challenge) throws -> Self {
let clientChallengeType: ClientChallenge.ChallengeType
switch challenge.type {
case .faceMovementAndLightChallenge:
clientChallengeType = .faceMovementAndLightChallenge(
challenge: .init(
challengeID: event.initialClientEvent.challengeID,
targetFace: .init(
boundingBox: .init(boundingBox: event.targetFace.initialEvent.boundingBox),
Expand All @@ -46,7 +48,26 @@ extension LivenessEvent where T == FinalClientEvent {
videoEndTimeStamp: Date().epochMilliseconds
)
)
)
case .faceMovementChallenge:
clientChallengeType = .faceMovementChallenge(
challenge: .init(
challengeID: event.initialClientEvent.challengeID,
targetFace: .init(
boundingBox: .init(boundingBox: event.targetFace.initialEvent.boundingBox),
faceDetectedInTargetPositionStartTimestamp: event.targetFace.initialEvent.startTimestamp,
faceDetectedInTargetPositionEndTimestamp: event.targetFace.endTimestamp
),
initialFace: .init(
boundingBox: .init(boundingBox: event.initialClientEvent.initialFaceLocation.boundingBox),
initialFaceDetectedTimeStamp: event.initialClientEvent.initialFaceLocation.startTimestamp
),
videoStartTimestamp: nil,
videoEndTimeStamp: Date().epochMilliseconds
)
)
}

let clientEvent = ClientSessionInformationEvent(challenge: .init(clientChallengeType: clientChallengeType))
let payload = try JSONEncoder().encode(clientEvent)
return .init(
payload: payload,
Expand Down
26 changes: 14 additions & 12 deletions AmplifyPlugins/Predictions/AWSPredictionsPlugin/Liveness/Events/LivenessFreshnessEvent.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,20 @@ extension LivenessEvent where T == FreshnessEvent {
public static func freshness(event: FreshnessEvent) throws -> Self {
let clientEvent = ClientSessionInformationEvent(
challenge: .init(
faceMovementAndLightChallenge: .init(
challengeID: event.challengeID,
targetFace: nil,
initialFace: nil,
videoStartTimestamp: nil,
colorDisplayed: .init(
currentColor: .init(rgb: event.color),
sequenceNumber: event.sequenceNumber,
currentColorStartTimeStamp: event.timestamp,
previousColor: .init(rgb: event.previousColor)
),
videoEndTimeStamp: nil
clientChallengeType: .faceMovementAndLightChallenge(
challenge: .init(
challengeID: event.challengeID,
targetFace: nil,
initialFace: nil,
videoStartTimestamp: nil,
colorDisplayed: .init(
currentColor: .init(rgb: event.color),
sequenceNumber: event.sequenceNumber,
currentColorStartTimeStamp: event.timestamp,
previousColor: .init(rgb: event.previousColor)
),
videoEndTimeStamp: nil
)
)
)
)
Expand Down
28 changes: 23 additions & 5 deletions ...Plugins/Predictions/AWSPredictionsPlugin/Liveness/Events/LivenessInitialClientEvent.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,20 @@ public struct InitialClientEvent {

extension LivenessEvent where T == InitialClientEvent {
@_spi(PredictionsFaceLiveness)
public static func initialFaceDetected(event: InitialClientEvent) throws -> Self {
public static func initialFaceDetected(
event: InitialClientEvent,
challenge: Challenge
) throws -> Self {
let initialFace = InitialFace(
boundingBox: .init(boundingBox: event.initialFaceLocation.boundingBox),
initialFaceDetectedTimeStamp: event.initialFaceLocation.startTimestamp
)

let clientSessionInformationEvent = ClientSessionInformationEvent(
challenge: .init(
faceMovementAndLightChallenge: .init(
let clientChallengeType: ClientChallenge.ChallengeType
switch challenge.type {
case .faceMovementAndLightChallenge:
clientChallengeType = .faceMovementAndLightChallenge(
challenge: .init(
challengeID: event.challengeID,
targetFace: nil,
initialFace: initialFace,
Expand All @@ -43,8 +48,21 @@ extension LivenessEvent where T == InitialClientEvent {
videoEndTimeStamp: nil
)
)
case .faceMovementChallenge:
clientChallengeType = .faceMovementChallenge(
challenge: .init(
challengeID: event.challengeID,
targetFace: nil,
initialFace: initialFace,
videoStartTimestamp: event.videoStartTimestamp,
videoEndTimeStamp: nil
)
)
}

let clientSessionInformationEvent = ClientSessionInformationEvent(
challenge: .init(clientChallengeType: clientChallengeType)
)

let payload = try JSONEncoder().encode(clientSessionInformationEvent)
return .init(
payload: payload,
Expand Down
93 changes: 56 additions & 37 deletions AmplifyPlugins/Predictions/AWSPredictionsPlugin/Liveness/Model/DTOMapping.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,18 @@
import Foundation

func ovalChallenge(from event: ServerSessionInformationEvent) -> FaceLivenessSession.OvalMatchChallenge {
let challengeConfig = event.sessionInformation.challenge.faceMovementAndLightChallenge.challengeConfig
let ovalParameters = event.sessionInformation.challenge.faceMovementAndLightChallenge.ovalParameters
let challengeConfig: ChallengeConfig
let ovalParameters: OvalParameters

switch event.sessionInformation.challenge.type {
case .faceMovementAndLightChallenge(let challenge):
challengeConfig = challenge.challengeConfig
ovalParameters = challenge.ovalParameters
case .faceMovementChallenge(let challenge):
challengeConfig = challenge.challengeConfig
ovalParameters = challenge.ovalParameters
}

let ovalBoundingBox = FaceLivenessSession.BoundingBox.init(
x: Double(ovalParameters.centerX - ovalParameters.width / 2),
y: Double(ovalParameters.centerY - ovalParameters.height / 2),
Expand Down Expand Up @@ -37,44 +47,46 @@ func ovalChallenge(from event: ServerSessionInformationEvent) -> FaceLivenessSes
)
}

func colorChallenge(from event: ServerSessionInformationEvent) -> FaceLivenessSession.ColorChallenge {
let displayColors = event.sessionInformation.challenge
.faceMovementAndLightChallenge.colorSequences
.map({ color -> FaceLivenessSession.DisplayColor in
func colorChallenge(from event: ServerSessionInformationEvent) -> FaceLivenessSession.ColorChallenge? {
switch event.sessionInformation.challenge.type {
case .faceMovementAndLightChallenge(let challenge):
let displayColors = challenge.colorSequences
.map({ color -> FaceLivenessSession.DisplayColor in

let duration: Double
let shouldScroll: Bool
switch (color.downscrollDuration, color.flatDisplayDuration) {
case (...0, 0...):
duration = Double(color.flatDisplayDuration)
shouldScroll = false
default:
duration = Double(color.downscrollDuration)
shouldScroll = true
}
let duration: Double
let shouldScroll: Bool
switch (color.downscrollDuration, color.flatDisplayDuration) {
case (...0, 0...):
duration = Double(color.flatDisplayDuration)
shouldScroll = false
default:
duration = Double(color.downscrollDuration)
shouldScroll = true
}

precondition(
color.freshnessColor.rgb.count == 3,
"""
Received invalid freshness colors.
Expected 3 values (r, g, b), received: \(color.freshnessColor.rgb.count)
"""
)
precondition(
color.freshnessColor.rgb.count == 3,
"""
Received invalid freshness colors.
Expected 3 values (r, g, b), received: \(color.freshnessColor.rgb.count)
"""
)

return .init(
rgb: .init(
red: Double(color.freshnessColor.rgb[0]) / 255,
green: Double(color.freshnessColor.rgb[1]) / 255,
blue: Double(color.freshnessColor.rgb[2]) / 255,
_values: color.freshnessColor.rgb
),
duration: duration,
shouldScroll: shouldScroll
)
})
return .init(
colors: displayColors
)
return .init(
rgb: .init(
red: Double(color.freshnessColor.rgb[0]) / 255,
green: Double(color.freshnessColor.rgb[1]) / 255,
blue: Double(color.freshnessColor.rgb[2]) / 255,
_values: color.freshnessColor.rgb
),
duration: duration,
shouldScroll: shouldScroll
)
})
return .init(colors: displayColors)
case .faceMovementChallenge:
return nil
}
}

func sessionConfiguration(from event: ServerSessionInformationEvent) -> FaceLivenessSession.SessionConfiguration {
Expand All @@ -83,3 +95,10 @@ func sessionConfiguration(from event: ServerSessionInformationEvent) -> FaceLive
ovalMatchChallenge: ovalChallenge(from: event)
)
}

func challengeType(from event: ChallengeEvent) -> Challenge {
.init(
version: event.version,
type: event.type
)
}
7 changes: 5 additions & 2 deletions ...ctions/AWSPredictionsPlugin/Liveness/Model/FaceLivenessSession+SessionConfiguration.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@ import Foundation
extension FaceLivenessSession {
@_spi(PredictionsFaceLiveness)
public struct SessionConfiguration {
public let colorChallenge: ColorChallenge
public let colorChallenge: ColorChallenge?
public let ovalMatchChallenge: OvalMatchChallenge

public init(colorChallenge: ColorChallenge, ovalMatchChallenge: OvalMatchChallenge) {
public init(
colorChallenge: ColorChallenge? = nil,
ovalMatchChallenge: OvalMatchChallenge
) {
self.colorChallenge = colorChallenge
self.ovalMatchChallenge = ovalMatchChallenge
}
Expand Down
14 changes: 12 additions & 2 deletions ...Plugins/Predictions/AWSPredictionsPlugin/Liveness/SPI/AWSPredictionsPlugin+Liveness.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ extension AWSPredictionsPlugin {
let session = FaceLivenessSession(
websocket: WebSocketSession(),
signer: signer,
baseURL: url
baseURL: url,
options: options
)

session.onServiceException = { completion(.failure($0)) }
Expand All @@ -48,7 +49,16 @@ extension AWSPredictionsPlugin {
extension FaceLivenessSession {
@_spi(PredictionsFaceLiveness)
public struct Options {
public init() {}
public let viewId: String
public let preCheckViewEnabled: Bool

public init(
faceLivenessDetectorViewId: String,
preCheckViewEnabled: Bool
) {
self.viewId = faceLivenessDetectorViewId
self.preCheckViewEnabled = preCheckViewEnabled
}
}
}

Expand Down
32 changes: 29 additions & 3 deletions AmplifyPlugins/Predictions/AWSPredictionsPlugin/Liveness/Service/FaceLivenessSession.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ public final class FaceLivenessSession: LivenessService {
let signer: SigV4Signer
let baseURL: URL
var serverEventListeners: [LivenessEventKind.Server: (FaceLivenessSession.SessionConfiguration) -> Void] = [:]
var challengeTypeListeners: [LivenessEventKind.Server: (Challenge) -> Void] = [:]
var onComplete: (ServerDisconnection) -> Void = { _ in }
let options: FaceLivenessSession.Options

private let livenessServiceDispatchQueue = DispatchQueue(
label: "com.amazon.aws.amplify.liveness.service",
Expand All @@ -26,12 +28,14 @@ public final class FaceLivenessSession: LivenessService {
init(
websocket: WebSocketSession,
signer: SigV4Signer,
baseURL: URL
baseURL: URL,
options: FaceLivenessSession.Options
) {
self.eventStreamEncoder = EventStream.Encoder()
self.eventStreamDecoder = EventStream.Decoder()
self.signer = signer
self.baseURL = baseURL
self.options = options

self.websocket = websocket

Expand All @@ -58,16 +62,27 @@ public final class FaceLivenessSession: LivenessService {
) {
serverEventListeners[event] = listener
}

public func register(listener: @escaping (Challenge) -> Void, on event: LivenessEventKind.Server) {
challengeTypeListeners[event] = listener
}

public func closeSocket(with code: URLSessionWebSocketTask.CloseCode) {
websocket.close(with: code)
}

public func initializeLivenessStream(withSessionID sessionID: String, userAgent: String = "") throws {
public func initializeLivenessStream(withSessionID sessionID: String,
userAgent: String = "",
challenges: [Challenge] = FaceLivenessSession.supportedChallenges) throws {
var components = URLComponents(url: baseURL, resolvingAgainstBaseURL: false)

components?.queryItems = [
URLQueryItem(name: "session-id", value: sessionID),
URLQueryItem(name: "challenge-versions", value: "FaceMovementAndLightChallenge_1.0.0"),
URLQueryItem(name: "precheck-view-enabled", value: options.preCheckViewEnabled ? "1":"0"),
// TODO: Change this after confirmation
URLQueryItem(name: "attempt-id", value: options.viewId),
URLQueryItem(name: "challenge-versions",
value: challenges.map({$0.queryParameterString()}).joined(separator: ",")),
URLQueryItem(name: "video-width", value: "480"),
URLQueryItem(name: "video-height", value: "640"),
URLQueryItem(name: "x-amz-user-agent", value: userAgent)
Expand Down Expand Up @@ -123,6 +138,9 @@ public final class FaceLivenessSession: LivenessService {
if let payload = try? JSONDecoder().decode(ServerSessionInformationEvent.self, from: message.payload) {
let sessionConfiguration = sessionConfiguration(from: payload)
self.serverEventListeners[.challenge]?(sessionConfiguration)
} else if let payload = try? JSONDecoder().decode(ChallengeEvent.self, from: message.payload) {
let challengeType = challengeType(from: payload)
self.challengeTypeListeners[.challenge]?(challengeType)
} else if (try? JSONDecoder().decode(DisconnectEvent.self, from: message.payload)) != nil {
onComplete(.disconnectionEvent)
return false
Expand All @@ -140,6 +158,14 @@ public final class FaceLivenessSession: LivenessService {
let serverEvent = LivenessEventKind.Server(rawValue: eventType.value)
switch serverEvent {
case .challenge:
// :event-type ChallengeEvent
let payload = try JSONDecoder().decode(
ChallengeEvent.self, from: message.payload
)
let challengeType = challengeType(from: payload)
challengeTypeListeners[.challenge]?(challengeType)
return true
case .sessionInformation:
// :event-type ServerSessionInformationEvent
let payload = try JSONDecoder().decode(
ServerSessionInformationEvent.self, from: message.payload
Expand Down
Loading

0 comments on commit 0148783

Please sign in to comment.