From 8d31193d8e3095e1032c0a5fdde2819e50d722a8 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Wed, 26 Jan 2022 02:27:42 +0900 Subject: [PATCH] kycdeepface integration --- package-lock.json | 29 ++ package.json | 1 + serverless-uncompiled.yml | 5 + src/in-house-bot/index.ts | 1 + src/in-house-bot/kycdeepface/index.ts | 122 ++++++ .../plugins/kycdeepface-checks.ts | 385 ++++++++++++++++++ 6 files changed, 543 insertions(+) create mode 100644 src/in-house-bot/kycdeepface/index.ts create mode 100644 src/in-house-bot/plugins/kycdeepface-checks.ts diff --git a/package-lock.json b/package-lock.json index a1d29255..4eba169e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1659,6 +1659,35 @@ "typeforce": "^1.0.0" } }, + "@tradle/urlsafe-base64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@tradle/urlsafe-base64/-/urlsafe-base64-1.0.0.tgz", + "integrity": "sha512-GXqIKpdeaWoVGfTJo7weCtD0+j54DmdH+HagPhLEPIm9lPa2feaUBkRePj0cvE7cD+Wttt3x6hm3twE1lstbKQ==", + "requires": { + "buffer": "^6.0.3" + }, + "dependencies": { + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + } + } + }, "@tradle/utils": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@tradle/utils/-/utils-1.3.0.tgz", diff --git a/package.json b/package.json index d0eca9d8..b9880caa 100644 --- a/package.json +++ b/package.json @@ -195,6 +195,7 @@ "@tradle/schema-graphql": "github:tradle/schema-graphql", "@tradle/schema-joi": "github:tradle/schema-joi", "@tradle/test-helpers": "github:tradle/test-helpers", + "@tradle/urlsafe-base64": "^1.0.0", "@tradle/validate-resource": "^4.3.5", "@tradle/web3-provider-engine": "^14.0.6", "JSONStream": "^1.3.5", diff --git a/serverless-uncompiled.yml b/serverless-uncompiled.yml index a6783e15..3b8f9def 100644 --- a/serverless-uncompiled.yml +++ b/serverless-uncompiled.yml @@ -744,6 +744,11 @@ functions: # method: put # cors: ${{self:custom.cors}} + kycdeepface: + image: public.ecr.aws/h7s8u8m7/kycdeepface:latest + timeout: 899 + memorySize: 1024 + # 1. generates temporary credentials (STS) for new connections, # and assumes IotClientRole on them # 2. creates an unauthenticated session, diff --git a/src/in-house-bot/index.ts b/src/in-house-bot/index.ts index a3925dd7..439daac1 100644 --- a/src/in-house-bot/index.ts +++ b/src/in-house-bot/index.ts @@ -770,6 +770,7 @@ export const loadComponentsAndPlugins = ({ 'leasingQuotes', 'giinCheck', 'vatCheck', + 'kycdeepface-checks', // 'invoicing' ].forEach((name) => attachPlugin({ name })) ;[ diff --git a/src/in-house-bot/kycdeepface/index.ts b/src/in-house-bot/kycdeepface/index.ts new file mode 100644 index 00000000..f6b36b89 --- /dev/null +++ b/src/in-house-bot/kycdeepface/index.ts @@ -0,0 +1,122 @@ +import { safe as b64 } from '@tradle/urlsafe-base64' +import fs from 'fs' +import { resolve } from 'path' +import debug from 'debug' + +const { readFile } = fs.promises +const log = debug('tradle:kycdeepface:index') + +export type Point = [ + number, + number +] + +export interface Embedding { + bounds: { + topLeft: Point, + bottomRight: Point + } + landmarks: { + outline: Point[] + left_brows: Point[] + right_brows: Point[] + nose_back: Point[] + nostrils: Point[] + left_eye: Point[] + right_eye: Point[] + mouth: Point[] + }, + angles: { + pitch: number + yaw: number + roll: number + } + embedding: string +} + +export interface Embeddings { + faces: Embedding[] + timings: { [type: string]: number } +} + +export interface Match { + similarity: number + timings: { [type: string]: number } +} + +export interface InputBytes { + image_bytes: Buffer +} + +export interface InputFile { + image_file: string +} + +export interface InputURL { + image_url: string +} + +export interface InputS3 { + image_s3: { + bucket: string + key: string + version?: string + } +} + +export interface InputBase64 { + image_urlsafe_b64: string +} + +export type Input = InputS3 | InputURL | InputBytes | InputFile | InputBase64 + +export function isInputFile (input: Input): input is InputFile { + return 'image_file' in input +} + +export function isInputBytes (input: Input): input is InputBytes { + return 'image_bytes' in input +} + +async function normalizeInput (input: Input): Promise { + if (isInputFile(input)) { + const pth = resolve(input.image_file) + try { + return normalizeInput({ + image_bytes: await readFile(pth) + }) + } catch (err) { + throw Object.assign(new Error(`Error while loading file ${pth}: ${err.message}`), err) + } + } + if (isInputBytes(input)) { + return { + image_urlsafe_b64: b64.encode(input.image_bytes) + } + } + return input +} + +export interface Exec { + description: string, + run: (input: any) => Promise +} + +async function exec(name: string, execFn: Exec, input: any): Promise { + log(name, execFn.description, input) + input = { [name]: input } + try { + return await execFn.run(input) + } catch (err) { + log(`${name}:retry`, err) + return await execFn.run(input) + } +} + +export async function face_embeddings (execFn: Exec, input: Input): Promise { + return await exec('face_embeddings', execFn, await normalizeInput(input)) +} + +export async function face_match (execFn: Exec, embedding_a: string, embedding_b: string): Promise { + return await exec('face_match', execFn, { embedding_a, embedding_b }) +} diff --git a/src/in-house-bot/plugins/kycdeepface-checks.ts b/src/in-house-bot/plugins/kycdeepface-checks.ts new file mode 100644 index 00000000..6b37ef96 --- /dev/null +++ b/src/in-house-bot/plugins/kycdeepface-checks.ts @@ -0,0 +1,385 @@ +import _ from 'lodash' + +import DataURI from 'strong-data-uri' + +import buildResource from '@tradle/build-resource' +import constants from '@tradle/constants' +import { + Bot, + Logger, + CreatePlugin, + Applications, + ITradleObject, + IPBApp, + IPBReq, + IPluginLifecycleMethods, + ValidatePluginConf, + Models +} from '../types' + +import { + getLatestForms, + doesCheckNeedToBeCreated, + // doesCheckExist, + getChecks, + hasPropertiesChanged, + getStatusMessageForCheck, + getThirdPartyServiceInfo +} from '../utils' + +import { face_embeddings, face_match, Embedding, Match, Exec } from '../kycdeepface' + +const { TYPE, TYPES } = constants +const { VERIFICATION } = TYPES +const SELFIE = 'tradle.Selfie' +const PHOTO_ID = 'tradle.PhotoID' +const FACIAL_RECOGNITION = 'tradle.FacialRecognitionCheck' +const ASPECTS = 'Face matching, Liveness detection' +const PROVIDER = 'KYCDeepFace' +const KYCDEEPFACE_API_RESOURCE = { + [TYPE]: 'tradle.API', + name: PROVIDER +} +const DEFAULT_THRESHOLD = 0.8 + +export const name = 'kycdeepface-checks' + +type KYCDeepFaceConf = { + endpoint: string + threshold: number +} + +interface RawData { + photoIdFace: Embedding + selfieFace: Embedding + match?: Match +} + +interface MatchResult { + status: 'error' | 'fail' | 'pass' + rawData?: RawData + error?: string +} + +function isMatchResult (input: object): input is MatchResult { + return 'status' in input +} + +async function getFace (execFn: Exec, models: Models, resource: ITradleObject, data: { url: string }): Promise { + const title = () => buildResource.title({ models, resource }) + const bytes = DataURI.decode(data.url) + return face_embeddings(execFn, { image_bytes: bytes }).then( + rawData => { + const { faces } = rawData + if (faces.length === 0) { + return { status: 'error', error: `No face found in "${title()}".` } + } + if (faces.length > 1) { + return { status: 'error', error: `More than one face (${faces.length}) found in "${title()}".` } + } + return rawData.faces[0] + }, + error => ({ status: 'error', error: `Couldnt extract faces from "${title()}: ${error.message}` }) + ) +} + +export class KYCDeepFaceAPI { + private bot: Bot + private logger: Logger + private applications: Applications + private conf: KYCDeepFaceConf + constructor({ bot, applications, logger, conf }) { + this.bot = bot + this.applications = applications + this.logger = logger + this.conf = conf + } + + public getSelfieAndPhotoID = async (application: IPBApp, req: IPBReq, payload: ITradleObject) => { + const stubs = getLatestForms(application) + + let isPhotoID = payload[TYPE] === PHOTO_ID + let rtype = isPhotoID ? SELFIE : PHOTO_ID + const stub = stubs.find(({ type }) => type === rtype) + if (!stub) { + // not enough info + return + } + this.logger.debug('Face recognition both selfie and photoId ready') + + const resource = await this.bot.getResource(stub) + let selfie, photoID + if (isPhotoID) { + photoID = payload + selfie = resource + } else { + photoID = resource + selfie = payload + } + let selfieLink = selfie._link + let photoIdLink = photoID._link + + let items + if (req.checks) { + items = req.checks.filter(r => r.provider === PROVIDER) + items.sort((a, b) => a.time - b.time) + } else { + items = await getChecks({ + bot: this.bot, + type: FACIAL_RECOGNITION, + application, + provider: PROVIDER + }) + } + + if (items.length) { + let checks = items.filter( + r => r.selfie._link === selfieLink || r.photoID._link === photoIdLink + ) + if (checks.length && checks[0].status.id !== 'tradle.Status_error') { + let check = checks[0] + if (check.selfie._link === selfieLink && check.photoID._link === photoIdLink) { + this.logger.debug( + `Rankone: check already exists for ${photoID.firstName} ${photoID.lastName} ${photoID.documentType.title}` + ) + return + } + // Check what changed photoID or Selfie. + // If it was Selfie then create a new check since Selfi is not editable + if (check.selfie._link === selfieLink) { + let changed = await hasPropertiesChanged({ + resource: photoID, + bot: this.bot, + propertiesToCheck: ['scan'], + req + }) + if (!changed) { + this.logger.debug( + `Rankone: nothing to check the 'scan' didn't change ${photoID.firstName} ${photoID.lastName} ${photoID.documentType.title}` + ) + return + } + } + } + } + await Promise.all([ + this.bot.resolveEmbeds(selfie), + this.bot.resolveEmbeds(photoID) + ]) + return { selfie, photoID } + } + + public async matchSelfieAndPhotoID ({ + selfie, + photoID + }: { + selfie: ITradleObject + photoID: ITradleObject + }): Promise { + const models = this.bot.models + // call whatever API with whatever params + const { threshold=DEFAULT_THRESHOLD } = this.conf + const execFn: Exec = { + description: `λ(kycdeepface)`, + run: (input) => this.bot.lambdaInvoker.invoke({ + name: 'kycdeepface', + arg: input + }) + } + + const [photoIdFace, selfieFace] = await Promise.all([ + getFace(execFn, models, photoID, photoID.rfidFace && photoID.rfidFace || photoID.scan), + getFace(execFn, models, selfie, selfie.selfie) + ]) + if (isMatchResult(photoIdFace)) { + return photoIdFace + } + if (isMatchResult(selfieFace)) { + return selfieFace + } + const rawData: RawData = { + photoIdFace, + selfieFace + } + try { + rawData.match = await face_match(execFn, photoIdFace.embedding, selfieFace.embedding) + this.logger.debug('Face recognition check, match:', rawData.match) + } catch (err) { + this.logger.error('Face recognition check error', err) + return { + status: 'error', + rawData, + error: `Could not compare faces in "${buildResource.title({ models, resource: photoID })}" and "${buildResource.title({ models, resource: selfie })}" : ${err.message}` + } + } + return { status: rawData.match.similarity > threshold ? 'pass' : 'fail', rawData } + } + public matchRfidFaceAndPhotoID = async (payload, application, req) => { + let createCheck = await doesCheckNeedToBeCreated({ + bot: this.bot, + type: FACIAL_RECOGNITION, + application, + provider: PROVIDER, + form: payload, + propertiesToCheck: ['scan'], + prop: 'form', + req + }) + if (!createCheck) return + const selfie = { [TYPE]: SELFIE, selfie: payload.rfidFace } + const match = await this.matchSelfieAndPhotoID({ + selfie, + photoID: payload + }) + await this.createCheck({ + match, + selfie, + photoID: payload, + application, + req + }) + } + public appendFileBuf = ({ form, filename, content, contentType }) => + form.append(filename, content, { filename, contentType }) + + public async createCheck ({ selfie, photoID, application, match, req }: { + selfie: ITradleObject + photoID: ITradleObject + application: IPBApp + match: MatchResult + req: IPBReq + }) { + const { models } = this.bot + const check = { + [TYPE]: FACIAL_RECOGNITION, + status: match.status, + provider: PROVIDER, + aspects: 'facial similarity', + rawData: match.rawData, + application, + form: selfie, + photoID, + score: match.rawData?.match?.similarity, + dateChecked: new Date().getTime(), + message: null as string + } + check.message = getStatusMessageForCheck({ models, check }) + + this.logger.debug( + `Creating KYCDeepFace ${FACIAL_RECOGNITION} for: ${photoID.firstName} ${photoID.lastName} (${buildResource.title({ models, resource: photoID })})` + ) + + return (await this.applications.createCheck(check, req)).toJSON() + } + + public createVerification = async ({ user, application, photoID, req, org }) => { + const method: any = { + [TYPE]: 'tradle.APIBasedVerificationMethod', + api: _.clone(KYCDEEPFACE_API_RESOURCE), + aspect: ASPECTS + } + + const verification = this.bot + .draft({ type: VERIFICATION }) + .set({ + document: photoID, + method + }) + .toJSON() + + await this.applications.createVerification({ + application, verification, org + }) + if (application.checks) + await this.applications.deactivateChecks({ + application, + type: FACIAL_RECOGNITION, + form: photoID, + req + }) + } +} + +const CONF_PROPERTY = 'kycdeepface' + +export const createPlugin: CreatePlugin = (components, pluginOpts) => { + const { bot, applications } = components + let { logger, conf = {} } = pluginOpts + + logger.info('kycd - 1 - setting up') + + const kycDeepFace = new KYCDeepFaceAPI({ + bot, + applications, + logger, + conf: { + ...getThirdPartyServiceInfo(components.conf, CONF_PROPERTY), + ...conf + } + }) + + const plugin: IPluginLifecycleMethods = { + async onmessage(req: IPBReq) { + if (req.skipChecks) return + const { user, application, payload } = req + logger.info('kycd 0', application) + if (!application) return + + let isPhotoID = payload[TYPE] === PHOTO_ID + let isSelfie = payload[TYPE] === SELFIE + logger.info(`kycd 1 ${isPhotoID}/${isSelfie}`) + if (!isPhotoID && !isSelfie) return + + logger.info('kycd 2 - getting stuff') + const result = await kycDeepFace.getSelfieAndPhotoID(application, req, payload) + if (!result) { + logger.info('kycd 3 - no result') + if (!isPhotoID || !payload.rfidFace) { + logger.info('kycd 4 - no photo, do nothing') + return + } + logger.info('kycd 5 - matchRfidFaceAndPhotoID') + kycDeepFace.matchRfidFaceAndPhotoID(payload, application, req) + return + } + logger.info('kycd 6 - matchSelfieAndPhotoID') + const { selfie, photoID } = result + const match = await kycDeepFace.matchSelfieAndPhotoID({ + selfie, + photoID + }) + logger.info('kycd 7', match) + const promiseCheck = kycDeepFace.createCheck({ + selfie, + photoID, + match, + application, + req + }) + const pchecks = [promiseCheck] + if (match.status === 'pass') { + const promiseVerification = kycDeepFace.createVerification({ + user, + application, + photoID, + req, + org: this.org + }) + pchecks.push(promiseVerification) + } + + await Promise.all(pchecks) + } + } + + return { + api: kycDeepFace, + plugin + } +} + +export const validateConf: ValidatePluginConf = async ({ conf, bot }) => { + // TODO: implement a test for the third party service + // ensureThirdPartyServiceConfigured(conf, CONF_PROPERTY) + bot.logger.info('Valid conf?', conf) +}