diff --git a/Demo/iOS/AICamera/AICamera/AICamera-Bridging-Header.h b/Demo/iOS/AICamera/AICamera/AICamera-Bridging-Header.h new file mode 100644 index 0000000..74b8357 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/AICamera-Bridging-Header.h @@ -0,0 +1,7 @@ +// +// Use this file to import your target's public headers that you would like to expose to Swift. +// + +#import "ImageRecognizerPaddleWrapper.h" +#import "PaddleHelper.h" +#import "SSDData.h" diff --git a/Demo/iOS/AICamera/AICamera/AppDelegate.swift b/Demo/iOS/AICamera/AICamera/AppDelegate.swift new file mode 100644 index 0000000..b524e79 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/AppDelegate.swift @@ -0,0 +1,49 @@ +// +// AppDelegate.swift +// SSDDemo +// +// Created by Nicky Chan on 11/6/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +import UIKit + +@UIApplicationMain +class AppDelegate: UIResponder, UIApplicationDelegate { + + var window: UIWindow? + + + func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplicationLaunchOptionsKey: Any]?) -> Bool { + //avoid screen off timer kicks off + UIApplication.shared.isIdleTimerDisabled = true + //init paddle libraries for one time + PaddleHelper.init_paddle() + return true + } + + func applicationWillResignActive(_ application: UIApplication) { + // Sent when the application is about to move from active to inactive state. This can occur for certain types of temporary interruptions (such as an incoming phone call or SMS message) or when the user quits the application and it begins the transition to the background state. + // Use this method to pause ongoing tasks, disable timers, and invalidate graphics rendering callbacks. Games should use this method to pause the game. + } + + func applicationDidEnterBackground(_ application: UIApplication) { + // Use this method to release shared resources, save user data, invalidate timers, and store enough application state information to restore your application to its current state in case it is terminated later. + // If your application supports background execution, this method is called instead of applicationWillTerminate: when the user quits. + } + + func applicationWillEnterForeground(_ application: UIApplication) { + // Called as part of the transition from the background to the active state; here you can undo many of the changes made on entering the background. + } + + func applicationDidBecomeActive(_ application: UIApplication) { + // Restart any tasks that were paused (or not yet started) while the application was inactive. If the application was previously in the background, optionally refresh the user interface. + } + + func applicationWillTerminate(_ application: UIApplication) { + // Called when the application is about to terminate. Save data if appropriate. See also applicationDidEnterBackground:. + } + + +} + diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/Contents.json b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000..a1cda81 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,106 @@ +{ + "images" : [ + { + "idiom" : "iphone", + "size" : "20x20", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "20x20", + "scale" : "3x" + }, + { + "size" : "29x29", + "idiom" : "iphone", + "filename" : "paddle-logo-58.png", + "scale" : "2x" + }, + { + "size" : "29x29", + "idiom" : "iphone", + "filename" : "paddle-logo-87.png", + "scale" : "3x" + }, + { + "idiom" : "iphone", + "size" : "40x40", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "40x40", + "scale" : "3x" + }, + { + "size" : "60x60", + "idiom" : "iphone", + "filename" : "paddle-logo-120.png", + "scale" : "2x" + }, + { + "size" : "60x60", + "idiom" : "iphone", + "filename" : "paddle-logo-180.png", + "scale" : "3x" + }, + { + "idiom" : "ipad", + "size" : "20x20", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "20x20", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "29x29", + "scale" : "1x" + }, + { + "size" : "29x29", + "idiom" : "ipad", + "filename" : "paddle-logo-59.png", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "40x40", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "40x40", + "scale" : "2x" + }, + { + "size" : "76x76", + "idiom" : "ipad", + "filename" : "paddle-logo-76.png", + "scale" : "1x" + }, + { + "size" : "76x76", + "idiom" : "ipad", + "filename" : "paddle-logo-152.png", + "scale" : "2x" + }, + { + "size" : "83.5x83.5", + "idiom" : "ipad", + "filename" : "paddle-logo-167.png", + "scale" : "2x" + }, + { + "idiom" : "ios-marketing", + "size" : "1024x1024", + "scale" : "1x" + } + ], + "info" : { + "version" : 1, + "author" : "xcode" + } +} \ No newline at end of file diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-120.png b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-120.png new file mode 100644 index 0000000..4d4ccf2 Binary files /dev/null and b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-120.png differ diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-152.png b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-152.png new file mode 100644 index 0000000..9771367 Binary files /dev/null and b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-152.png differ diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-167.png b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-167.png new file mode 100644 index 0000000..353cd00 Binary files /dev/null and b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-167.png differ diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-180.png b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-180.png new file mode 100644 index 0000000..b45c623 Binary files /dev/null and b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-180.png differ diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-58.png b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-58.png new file mode 100644 index 0000000..31d4b7e Binary files /dev/null and b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-58.png differ diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-59.png b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-59.png new file mode 100644 index 0000000..31d4b7e Binary files /dev/null and b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-59.png differ diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-76.png b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-76.png new file mode 100644 index 0000000..ef836a5 Binary files /dev/null and b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-76.png differ diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-87.png b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-87.png new file mode 100644 index 0000000..8e2f398 Binary files /dev/null and b/Demo/iOS/AICamera/AICamera/Assets.xcassets/AppIcon.appiconset/paddle-logo-87.png differ diff --git a/Demo/iOS/AICamera/AICamera/Assets.xcassets/Contents.json b/Demo/iOS/AICamera/AICamera/Assets.xcassets/Contents.json new file mode 100644 index 0000000..da4a164 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "version" : 1, + "author" : "xcode" + } +} \ No newline at end of file diff --git a/Demo/iOS/AICamera/AICamera/Base.lproj/LaunchScreen.storyboard b/Demo/iOS/AICamera/AICamera/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 0000000..fdf3f97 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Demo/iOS/AICamera/AICamera/Base.lproj/Main.storyboard b/Demo/iOS/AICamera/AICamera/Base.lproj/Main.storyboard new file mode 100644 index 0000000..a3c5279 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/Base.lproj/Main.storyboard @@ -0,0 +1,149 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Demo/iOS/AICamera/AICamera/ImageRecognizer.swift b/Demo/iOS/AICamera/AICamera/ImageRecognizer.swift new file mode 100644 index 0000000..18e197a --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/ImageRecognizer.swift @@ -0,0 +1,33 @@ +// +// ImageRecognizer.swift +// SSDDemo +// +// Created by Nicky Chan on 11/7/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +import Foundation + +protocol ImageRecognizerDelegate { + func imageRecognizedSuccess(_ result: SSDData) + func imageRecognizedError() +} + +class ImageRecognizer { + + var imageRecognizer: ImageRecognizerPaddleWrapper? + + init(model: SSDModel) { + imageRecognizer = ImageRecognizerPaddleWrapper(model: model.rawValue, withNormHeight: model.normDimension().0, withNormWidth: model.normDimension().1) + } + + func inference(imageBuffer: UnsafeMutablePointer!, width: Int32, height: Int32, score: Float) -> NSMutableArray! { + + return imageRecognizer?.inference(imageBuffer, withHeight: height, withWidth: width, withFilterScore: score) + } + + func release() { + imageRecognizer?.destroy() + } + +} \ No newline at end of file diff --git a/Demo/iOS/AICamera/AICamera/ImageRecognizerPaddleWrapper.h b/Demo/iOS/AICamera/AICamera/ImageRecognizerPaddleWrapper.h new file mode 100644 index 0000000..a330c8c --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/ImageRecognizerPaddleWrapper.h @@ -0,0 +1,22 @@ +// +// ImageRecognizerPaddleWrapper.h +// AICamera +// +// Created by Nicky Chan on 11/9/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +#ifndef ImageRecognizerPaddleWrapper_h +#define ImageRecognizerPaddleWrapper_h + +#import + +@interface ImageRecognizerPaddleWrapper : NSObject + +- (instancetype)initWithModel:(NSString*)modelFileName withNormHeight:(int)height withNormWidth:(int)width; +- (NSMutableArray*)inference:(unsigned char *)pixels withHeight:(int)height withWidth:(int)width withFilterScore:(float) filterScore; +- (void)destroy; + +@end + +#endif /* ImageRecognizerPaddleWrapper_h */ diff --git a/Demo/iOS/AICamera/AICamera/ImageRecognizerPaddleWrapper.mm b/Demo/iOS/AICamera/AICamera/ImageRecognizerPaddleWrapper.mm new file mode 100644 index 0000000..0d829d6 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/ImageRecognizerPaddleWrapper.mm @@ -0,0 +1,78 @@ +// +// ImageRecognizerPaddleWrapper.m +// SSDDemo +// +// Created by Nicky Chan on 11/7/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +#import +#import "ImageRecognizerPaddleWrapper.h" +#include "paddle_image_recognizer.h" +#include "SSDData.h" + +@interface ImageRecognizerPaddleWrapper () { + ImageRecognizer recognizer; +} +@end + +@implementation ImageRecognizerPaddleWrapper + +static NSString * kLabels [21] = { + @"background", @"aeroplane", @"bicycle", @"background", @"boat", + @"bottle", @"bus", @"car", @"cat", @"chair", + @"cow", @"diningtable", @"dog", @"horse", @"motorbike", + @"person", @"pottedplant", @"sheep", @"sofa", @"train", + @"tvmonitor" +}; + +- (instancetype)initWithModel:(NSString*)modelFileName withNormHeight:(int)height withNormWidth:(int)width { + self = [super init]; + if (self) + { + int channel = 3; + const std::vector means({104, 117, 124}); + + NSBundle* bundle = [NSBundle mainBundle]; + NSString* resourceDirectoryPath = [bundle bundlePath]; + NSString* path = [[resourceDirectoryPath stringByAppendingString:@"/"] stringByAppendingString:modelFileName]; + + self->recognizer.init([path UTF8String], height, width, channel, means); + + } + return self; +} + +- (NSMutableArray*)inference:(unsigned char *)pixels withHeight:(int)height withWidth:(int)width withFilterScore:(float) filterScore{ + ImageRecognizer::Result result; + int channel = 4; + image::Config config(image::kBGR, image::CLOCKWISE_R90); + self->recognizer.infer(pixels, height, width, channel, config, result); + + NSMutableArray *array = [[NSMutableArray alloc] initWithCapacity:result.height]; + int w = result.width; + + for (int i = 0; i < result.height; i++) { + float score = result.data[i * w + 2]; + if (score < filterScore) continue; + + SSDData *ssdData = [[SSDData alloc] init]; + ssdData.label = kLabels[(int) result.data[i * w + 1]]; + ssdData.accuracy = score; + ssdData.xmin = result.data[i * w + 3]; + ssdData.ymin = result.data[i * w + 4]; + ssdData.xmax = result.data[i * w + 5]; + ssdData.ymax = result.data[i * w + 6]; + + [array addObject:ssdData]; + } + + return array; +} + +- (void)destroy { + self->recognizer.release(); +} + + +@end diff --git a/Demo/iOS/AICamera/AICamera/Info.plist b/Demo/iOS/AICamera/AICamera/Info.plist new file mode 100644 index 0000000..10adb41 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/Info.plist @@ -0,0 +1,51 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + PDCamera + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + NSCameraUsageDescription + Requires camera access + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile + Main + UIRequiredDeviceCapabilities + + armv7 + + UIRequiresFullScreen + + UIStatusBarHidden + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/Demo/iOS/AICamera/AICamera/PaddleHelper.h b/Demo/iOS/AICamera/AICamera/PaddleHelper.h new file mode 100644 index 0000000..253bf19 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/PaddleHelper.h @@ -0,0 +1,16 @@ +// +// PaddleHelper.h +// AICamera +// +// Created by Nicky Chan on 11/13/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +#ifndef PaddleHelper_h +#define PaddleHelper_h + +@interface PaddleHelper : NSObject ++ (void)init_paddle; +@end + +#endif /* PaddleHelper_h */ diff --git a/Demo/iOS/AICamera/AICamera/PaddleHelper.mm b/Demo/iOS/AICamera/AICamera/PaddleHelper.mm new file mode 100644 index 0000000..0a7ac54 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/PaddleHelper.mm @@ -0,0 +1,19 @@ +// +// PaddleHelper.m +// AICamera +// +// Created by Nicky Chan on 11/13/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +#import +#include "paddle_image_recognizer.h" +#include "PaddleHelper.h" + +@implementation PaddleHelper : NSObject + ++ (void)init_paddle { + ImageRecognizer::init_paddle(); +} + +@end diff --git a/Demo/iOS/AICamera/AICamera/SSDData.h b/Demo/iOS/AICamera/AICamera/SSDData.h new file mode 100644 index 0000000..2745648 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/SSDData.h @@ -0,0 +1,20 @@ +// +// SSDData.h +// AICamera +// +// Created by Nicky Chan on 11/11/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +#import + +@interface SSDData : NSObject + +@property (nonatomic) NSString *label; +@property (nonatomic) float accuracy; +@property (nonatomic) float xmin; +@property (nonatomic) float ymin; +@property (nonatomic) float xmax; +@property (nonatomic) float ymax; + +@end \ No newline at end of file diff --git a/Demo/iOS/AICamera/AICamera/SSDData.m b/Demo/iOS/AICamera/AICamera/SSDData.m new file mode 100644 index 0000000..4ac6b58 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/SSDData.m @@ -0,0 +1,20 @@ +// +// SSDData.m +// AICamera +// +// Created by Nicky Chan on 11/11/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +#import +#import "SSDData.h" + +@implementation SSDData + +- (id)init +{ + self = [super init]; + return self; +} + +@end diff --git a/Demo/iOS/AICamera/AICamera/SSDDrawLayer.swift b/Demo/iOS/AICamera/AICamera/SSDDrawLayer.swift new file mode 100644 index 0000000..6dfb651 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/SSDDrawLayer.swift @@ -0,0 +1,62 @@ +// +// SSDDrawLayer.swift +// SSDDemo +// +// Created by Nicky Chan on 11/7/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +import UIKit + +class SSDDrawLayer: CAShapeLayer { + var labelLayer = CATextLayer() + + required override init() { + super.init() + } + + required init?(coder aDecoder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + func render(_ data: SSDData, model:SSDModel, isBackCamera:Bool) { + + let screenWidth = UIScreen.main.bounds.size.width + let screenHeight = UIScreen.main.bounds.size.height + + let x = CGFloat(isBackCamera ? data.xmin : 1 - data.xmax) * screenWidth + let y = CGFloat(data.ymin) * screenHeight + let width = CGFloat(data.xmax - data.xmin) * screenWidth + let height = CGFloat(data.ymax - data.ymin) * screenHeight + + if (model == SSDModel.FaceMobileNet160 && data.label != "aeroplane") { + return; + } + + //draw box + self.path = UIBezierPath(roundedRect: CGRect(x: x, y: y, width: width, height: height), cornerRadius: 10).cgPath + self.strokeColor = UIColor.cyan.cgColor + self.lineWidth = 4.0 + self.fillColor = nil + self.lineJoin = kCALineJoinBevel + + if (model == SSDModel.FaceMobileNet160) { + //do not draw label for face + return; + } + + let text = String.init(format: "%@: %.02f", data.label, data.accuracy) + var displayString = NSAttributedString(string: text, attributes: [ + NSStrokeColorAttributeName : UIColor.black, + NSForegroundColorAttributeName : UIColor.white, + NSStrokeWidthAttributeName : NSNumber(value: -6.0), + NSFontAttributeName : UIFont.systemFont(ofSize: 20, weight: 3) + ]) + + //draw label + + labelLayer.string = displayString + labelLayer.frame = CGRect.init(x: x + 4, y: y + height - 22, width: 1000, height: 30) + addSublayer(labelLayer) + } +} diff --git a/Demo/iOS/AICamera/AICamera/SSDModel.swift b/Demo/iOS/AICamera/AICamera/SSDModel.swift new file mode 100644 index 0000000..711b2dc --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/SSDModel.swift @@ -0,0 +1,29 @@ +// +// SSDModel.swift +// AICamera +// +// Created by Nicky Chan on 11/14/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +import Foundation + +enum SSDModel : String { + case PascalMobileNet300 = "pascal_mobilenet_300_66.paddle" + case FaceMobileNet160 = "face_mobilenet_160_91.paddle" + case PascalVGG300 = "vgg_ssd_net.paddle" + + func normDimension() -> (Int32, Int32) + { + switch self + { + case .PascalMobileNet300: + return (300, 300) + case .FaceMobileNet160: + return (160, 160) + case .PascalVGG300: + return (300, 300) + } + } +} + diff --git a/Demo/iOS/AICamera/AICamera/SSDMultiboxLayer.swift b/Demo/iOS/AICamera/AICamera/SSDMultiboxLayer.swift new file mode 100644 index 0000000..3d17e21 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/SSDMultiboxLayer.swift @@ -0,0 +1,25 @@ +// +// SSDMultiboxLayer.swift +// SSDDemo +// +// Created by Wang,Jeff on 11/7/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +import UIKit + +class SSDMultiboxLayer: CALayer { + + func displayBoxs(with ssdDataList: NSMutableArray, model: SSDModel, isBackCamera: Bool){ + self.sublayers?.forEach({ (layer) in + layer.removeFromSuperlayer() + }) + + for ssdData in ssdDataList { + let boxLayer = SSDDrawLayer.init() + boxLayer.render(ssdData as! SSDData, model: model, isBackCamera: isBackCamera) + + self.addSublayer(boxLayer) + } + } +} diff --git a/Demo/iOS/AICamera/AICamera/ViewController.swift b/Demo/iOS/AICamera/AICamera/ViewController.swift new file mode 100644 index 0000000..9ee0d99 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/ViewController.swift @@ -0,0 +1,315 @@ +// +// ViewController.swift +// SSDDemo +// +// Created by Nicky Chan on 11/6/17. +// Copyright © 2017 PaddlePaddle. All rights reserved. +// + +import UIKit +import AVFoundation +import Foundation + + +class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate { + + var captureSession : AVCaptureSession? + var multiboxLayer : SSDMultiboxLayer? + var previewLayer : AVCaptureVideoPreviewLayer? + var captureDevice : AVCaptureDevice? + + var isRestarting = false; + + var imageRecognizer : ImageRecognizer? + + var timeStamp : TimeInterval? + + var index = 0 + + //default settings + var ssdModel : SSDModel = SSDModel.PascalMobileNet300 + var accuracyThreshold : Float = 0.5 + var minTimeInterval : Float = 0.3 + var backCamera = true + + @IBOutlet weak var settingsView: UIView! + + @IBOutlet weak var accuracyLabel: UILabel! + @IBOutlet weak var timeRefreshLabel: UILabel! + @IBOutlet weak var pascalMobileNetBtn: UIButton! + @IBOutlet weak var pascalVgg300Btn: UIButton! + @IBOutlet weak var faceMobileNetBtn: UIButton! + @IBOutlet weak var backCameraBtn: UIButton! + @IBOutlet weak var frontCameraBtn: UIButton! + @IBOutlet weak var accuracySlider: UISlider! + @IBOutlet weak var timeRefreshSlider: UISlider! + + + @IBAction func pascalMobileNet300Click(_ sender: UIButton) { + pendingRestartWithNewModel(newModel: SSDModel.PascalMobileNet300) + } + + @IBAction func faceMobileNet300Click(_ sender: UIButton) { + pendingRestartWithNewModel(newModel: SSDModel.FaceMobileNet160) + } + + @IBAction func pascalVgg300Click(_ sender: UIButton) { + pendingRestartWithNewModel(newModel: SSDModel.PascalVGG300) + } + + @IBAction func backCameraClick(_ sender: UIButton) { + pendingRestartWithCamera(backCamera: true) + } + + @IBAction func frontCameraClick(_ sender: UIButton) { + pendingRestartWithCamera(backCamera: false) + } + + @IBAction func accurcyThresholdChanged(_ sender: UISlider) { + + accuracyThreshold = sender.value + accuracyLabel.text = String.init(format: "%.02f", accuracyThreshold) + let defaults = UserDefaults.standard + defaults.set(accuracyThreshold, forKey: "accuracyThreshold") + } + + @IBAction func timeRefreshChanged(_ sender: UISlider) { + + minTimeInterval = sender.value + timeRefreshLabel.text = String.init(format: "%.02f", minTimeInterval) + let defaults = UserDefaults.standard + defaults.set(minTimeInterval, forKey: "timeRefresh") + } + + func pendingRestartWithNewModel(newModel: SSDModel) { + + if ssdModel == newModel { + return; + } + + let defaults = UserDefaults.standard + defaults.set(newModel.rawValue , forKey: "model") + + isRestarting = true + ssdModel = newModel + } + + + func pendingRestartWithCamera(backCamera: Bool) { + + if self.backCamera == backCamera { + return; + } + + let defaults = UserDefaults.standard + defaults.set(backCamera , forKey: "backCamera") + + isRestarting = true + self.backCamera = backCamera + } + + func restart() { + //hack: just make it crash so that we can restart + exit(0) + DispatchQueue.main.async { + self.timeStamp = nil + self.index = 0; + + self.imageRecognizer?.release() + self.imageRecognizer = ImageRecognizer(model: self.ssdModel) + + self.captureSession?.stopRunning() + + self.previewLayer?.removeFromSuperlayer() + self.multiboxLayer?.removeFromSuperlayer() + self.setupVideoCaptureAndStart() + + self.isRestarting = false + } + } + + func toggleSettings(_ sender:UITapGestureRecognizer){ + settingsView.isHidden = !settingsView.isHidden + } + + override func viewDidLoad() { + super.viewDidLoad() + + self.settingsView.isHidden = true + + checkModel() + + populateInitialSettings() + + let gesture = UITapGestureRecognizer(target: self, action: #selector (self.toggleSettings (_:))) + self.view.addGestureRecognizer(gesture) + + imageRecognizer = ImageRecognizer(model: ssdModel) + + setupVideoCaptureAndStart() + } + + func checkModel() { + var bundlePath = Bundle.main.bundlePath + bundlePath.append("/") + bundlePath.append(SSDModel.PascalVGG300.rawValue) + pascalVgg300Btn.isHidden = !FileManager.default.fileExists(atPath: bundlePath) + } + + func populateInitialSettings() { + + let defaults = UserDefaults.standard + + if let modelStr = defaults.string(forKey:"model") { + self.ssdModel = SSDModel(rawValue: modelStr)! + } + var highlightBtn : UIButton? + if ssdModel == SSDModel.FaceMobileNet160 { + highlightBtn = faceMobileNetBtn; + } else if ssdModel == SSDModel.PascalMobileNet300 { + highlightBtn = pascalMobileNetBtn; + } else if ssdModel == SSDModel.PascalVGG300 { + highlightBtn = pascalVgg300Btn; + } + highlightBtn?.titleLabel?.font = UIFont.boldSystemFont(ofSize: 16) + highlightBtn?.setTitleColor(self.view.tintColor, for: .normal) + + if let backCamera = defaults.object(forKey: "backCamera") { + self.backCamera = backCamera as! Bool + } + + if self.backCamera { + backCameraBtn.titleLabel?.font = UIFont.boldSystemFont(ofSize: 16) + backCameraBtn.setTitleColor(self.view.tintColor, for: .normal) + } else { + frontCameraBtn.titleLabel?.font = UIFont.boldSystemFont(ofSize: 16) + frontCameraBtn.setTitleColor(self.view.tintColor, for: .normal) + } + + if let accuracyThreshold = defaults.object(forKey: "accuracyThreshold") { + self.accuracyThreshold = accuracyThreshold as! Float + accuracySlider.setValue(self.accuracyThreshold, animated: false) + } + + if let timeRefresh = defaults.object(forKey: "timeRefresh") { + self.minTimeInterval = timeRefresh as! Float + timeRefreshSlider.setValue(self.minTimeInterval, animated: false) + } + + accuracyLabel.text = String.init(format: "%.02f", accuracyThreshold) + timeRefreshLabel.text = String.init(format: "%.02f", minTimeInterval) + + } + + func setupVideoCaptureAndStart() { + + captureSession = AVCaptureSession() + if let captureSession = captureSession { + captureSession.sessionPreset = AVCaptureSessionPresetHigh + + captureDevice = AVCaptureDeviceDiscoverySession(deviceTypes: [AVCaptureDeviceType.builtInWideAngleCamera], mediaType: AVMediaTypeVideo, position: backCamera ? AVCaptureDevicePosition.back : AVCaptureDevicePosition.front).devices.first + + // setup video device input + do { + let videoDeviceInput: AVCaptureDeviceInput + do { + videoDeviceInput = try AVCaptureDeviceInput(device: captureDevice) + } + catch { + fatalError("Could not create AVCaptureDeviceInput instance with error: \(error).") + } + + captureSession.beginConfiguration() + guard captureSession.canAddInput(videoDeviceInput) else { + fatalError("CaptureSession can not add input") + } + captureSession.addInput(videoDeviceInput) + } + + // setup preview + let previewContainer = self.view.layer + let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)! + previewLayer.frame = previewContainer.bounds + previewLayer.contentsGravity = kCAGravityResizeAspect + previewLayer.videoGravity = AVLayerVideoGravityResizeAspect + previewContainer.insertSublayer(previewLayer, at: 0) + self.previewLayer = previewLayer + + multiboxLayer = SSDMultiboxLayer() + previewContainer.insertSublayer(multiboxLayer!, at: 1) + + // setup video output + do { + let videoDataOutput = AVCaptureVideoDataOutput() + videoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey: Int(kCVPixelFormatType_32BGRA)] + videoDataOutput.alwaysDiscardsLateVideoFrames = true + guard captureSession.canAddOutput(videoDataOutput) else { + fatalError("CaptureSession can not add output") + } + captureSession.addOutput(videoDataOutput) + + captureSession.commitConfiguration() + + let queue = DispatchQueue(label: "com.paddlepaddle.SSDDemo") + videoDataOutput.setSampleBufferDelegate(self, queue: queue) + + if let connection = videoDataOutput.connection(withMediaType: AVMediaTypeVideo) { + if connection.isVideoOrientationSupported { + // Force recording to portrait + // use portrait does not work for some reason, try to rotate in c++ code instead + // connection.videoOrientation = .portrait + } + } + captureSession.startRunning() + } + } + } + + func captureOutput(_ output: AVCaptureOutput, didDrop sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { + } + + func captureOutput(_ output: AVCaptureOutput, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { + + if let ts = self.timeStamp { + while(true) { + if (NSDate().timeIntervalSince1970 >= Double(minTimeInterval) + ts) { + break; + } + } + } + + index = index + 1 + if let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) { + CVPixelBufferLockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0)) + + let width = CVPixelBufferGetWidth(imageBuffer) + let height = CVPixelBufferGetHeight(imageBuffer) + let baseAddress = CVPixelBufferGetBaseAddress(imageBuffer) + + let intBuffer = unsafeBitCast(baseAddress, to: UnsafeMutablePointer.self) + + CVPixelBufferUnlockBaseAddress(imageBuffer, CVPixelBufferLockFlags(rawValue: 0)) + + let ssdDataList = imageRecognizer?.inference(imageBuffer: intBuffer, width: Int32(width), height: Int32(height), score: accuracyThreshold) + + self.timeStamp = NSDate().timeIntervalSince1970 + + DispatchQueue.main.async { + self.multiboxLayer?.displayBoxs(with: ssdDataList!, model:self.ssdModel, isBackCamera:self.backCamera) + } + } + + if (isRestarting) { + restart() + } + } + + + override func didReceiveMemoryWarning() { + super.didReceiveMemoryWarning() + + print("didReceiveMemoryWarning") + // Dispose of any resources that can be recreated. + } + +} diff --git a/Demo/iOS/AICamera/AICamera/image.h b/Demo/iOS/AICamera/AICamera/image.h new file mode 100644 index 0000000..fe6553a --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/image.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License */ + +#pragma once + +namespace image { + + enum Order { kCHW = 0, kHWC = 1 }; + + enum Format { + kRGB = 0x1, // support RGB, RGBA + kBGR = 0x2 // support BGR, BGRA + }; + + enum RotateOption { + NO_ROTATE = 0, + CLOCKWISE_R90 = 1, + CLOCKWISE_R180 = 2, + CLOCKWISE_R270 = 3 + }; + + struct Config { + Config() : format(kRGB), option(NO_ROTATE) {} + Config(Format f, RotateOption o) : format(f), option(o) {} + Format format; + RotateOption option; + }; + +} // namespace image diff --git a/Demo/iOS/AICamera/AICamera/image_utils.cpp b/Demo/iOS/AICamera/AICamera/image_utils.cpp new file mode 100644 index 0000000..bca6100 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/image_utils.cpp @@ -0,0 +1,149 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License */ + +#include "image_utils.h" +#include +#include + +namespace image { +namespace utils { + +void resize_hwc(const unsigned char* pixels, + unsigned char* resized_pixels, + const size_t height, + const size_t width, + const size_t channel, + const size_t resized_height, + const size_t resized_width) { + float ratio_x = static_cast(width) / static_cast(resized_width); + float ratio_y = + static_cast(height) / static_cast(resized_height); + + for (size_t i = 0; i < resized_height; i++) { + float new_y = i * ratio_y; + + size_t y1 = (static_cast(new_y) > (height - 1)) + ? (height - 1) + : static_cast(new_y); + size_t y2 = y1 + 1; + + float b1 = y2 - new_y; + float b2 = new_y - y1; + + for (size_t j = 0; j < resized_width; j++) { + float new_x = j * ratio_x; + + size_t x1 = (static_cast(new_x) > (width - 1)) + ? (width - 1) + : static_cast(new_x); + int x2 = x1 + 1; + + float a1 = x2 - new_x; + float a2 = new_x - x1; + + unsigned char* pt_dst = + resized_pixels + (i * resized_width + j) * channel; + const unsigned char* pt_src = pixels + (y1 * width + x1) * channel; + int p1 = width * channel; + int p2 = p1 + channel; + + if (x1 == width - 1 && y1 == height - 1) { + memcpy(pt_dst, pt_src, channel * sizeof(unsigned char)); + } else if (x1 == width - 1) { + for (size_t k = 0; k < channel; k++) { + float pixel_00 = static_cast(pt_src[k]); + float pixel_10 = static_cast(pt_src[p1 + k]); + + pt_dst[k] = static_cast(pixel_00 * b1 + pixel_10 * b2); + } + } else if (y1 == height - 1) { + for (size_t k = 0; k < channel; k++) { + float pixel_00 = static_cast(pt_src[k]); + float pixel_01 = static_cast(pt_src[channel + k]); + + pt_dst[k] = static_cast(pixel_00 * a1 + pixel_01 * a2); + } + } else { + // If x1 = width - 1 or y1 = height - 1, the memory accesses may be out + // of range. + for (size_t k = 0; k < channel; k++) { + float pixel_00 = static_cast(pt_src[k]); + float pixel_01 = static_cast(pt_src[channel + k]); + float pixel_10 = static_cast(pt_src[p1 + k]); + float pixel_11 = static_cast(pt_src[p2 + k]); + + pt_dst[k] = + static_cast((pixel_00 * a1 + pixel_01 * a2) * b1 + + (pixel_10 * a1 + pixel_11 * a2) * b2); + } + } + } // j-loop + } // i-loop +} + +void rotate_hwc(const unsigned char* pixels, + unsigned char* rotated_pixels, + const size_t height, + const size_t width, + const size_t channel, + const RotateOption option) { + switch (option) { + case NO_ROTATE: + memcpy(rotated_pixels, + pixels, + height * width * channel * sizeof(unsigned char)); + break; + case CLOCKWISE_R90: + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + // (i, j) -> (j, height - 1 - i) + for (size_t k = 0; k < channel; ++k) { + rotated_pixels[(j * height + height - 1 - i) * channel + k] = + pixels[(i * width + j) * channel + k]; + } + } + } + break; + case CLOCKWISE_R180: + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + // (i, j) -> (height - 1 - i, width - 1 - j) + for (size_t k = 0; k < channel; ++k) { + rotated_pixels[((height - 1 - i) * width + width - 1 - j) * + channel + + k] = pixels[(i * width + j) * channel + k]; + } + } + } + break; + case CLOCKWISE_R270: + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + // (i, j) -> (width - 1 - j, i) + for (size_t k = 0; k < channel; ++k) { + rotated_pixels[((width - 1 - j) * height + i) * channel + k] = + pixels[(i * width + j) * channel + k]; + } + } + } + break; + default: + fprintf(stderr, + "Illegal rotate option, please specify among [NO_ROTATE, " + "CLOCKWISE_R90, CLOCKWISE_R180, CLOCKWISE_R270].\n"); + } +} + +} // namespace utils +} // namespace image diff --git a/Demo/iOS/AICamera/AICamera/image_utils.h b/Demo/iOS/AICamera/AICamera/image_utils.h new file mode 100644 index 0000000..951fb15 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/image_utils.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License */ + +#pragma once + +#include +#include "image.h" + +namespace image { +namespace utils { + +void resize_hwc(const unsigned char* pixels, + unsigned char* resized_pixels, + const size_t height, + const size_t width, + const size_t channel, + const size_t resized_height, + const size_t resized_width); + +void rotate_hwc(const unsigned char* pixels, + unsigned char* rotated_pixels, + const size_t height, + const size_t width, + const size_t channel, + const RotateOption option); + +} // namespace utils +} // namespace image diff --git a/Demo/iOS/AICamera/AICamera/paddle_image_recognizer.cpp b/Demo/iOS/AICamera/AICamera/paddle_image_recognizer.cpp new file mode 100644 index 0000000..15c05f3 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/paddle_image_recognizer.cpp @@ -0,0 +1,230 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License */ + +#include "paddle_image_recognizer.h" +#include + +static void* read_config(const char* filename, long* size) { + FILE* file = fopen(filename, "rb"); + if (file == NULL) { + fprintf(stderr, "Open %s error\n", filename); + return NULL; + } + fseek(file, 0L, SEEK_END); + *size = ftell(file); + fseek(file, 0L, SEEK_SET); + void* buf = malloc(*size); + fread(buf, 1, *size, file); + fclose(file); + return buf; +} + +void ImageRecognizer::init(const char* merged_model_path, + const size_t normed_height, + const size_t normed_width, + const size_t normed_channel, + const std::vector& means) { + // Set the normed image size + normed_height_ = normed_height; + normed_width_ = normed_width; + normed_channel_ = normed_channel; + + // Set means + if (!means.empty()) { + means_ = means; + } else { + means_.clear(); + for (size_t i = 0; i < normed_channel; ++i) { + means_.push_back(0.0f); + } + } + + // Step 1: Reading merged model. + long size; + void* buf = read_config(merged_model_path, &size); + + // Step 2: + // Create a gradient machine for inference. + CHECK(paddle_gradient_machine_create_for_inference_with_parameters( + &gradient_machine_, buf, size)); + + free(buf); + buf = nullptr; +} + +void ImageRecognizer::preprocess(const unsigned char* pixels, + float* normed_pixels, + const size_t height, + const size_t width, + const size_t channel, + const image::Config& config) { + bool need_resize = true; + size_t resized_height = 0; + size_t resized_width = 0; + if (config.option == image::NO_ROTATE || + config.option == image::CLOCKWISE_R180) { + if (height == normed_height_ && width == normed_width_) { + need_resize = false; + } + resized_height = normed_height_; + resized_width = normed_width_; + } else if (config.option == image::CLOCKWISE_R90 || + config.option == image::CLOCKWISE_R270) { + if (height == normed_width_ && width == normed_height_) { + need_resize = false; + } + resized_height = normed_width_; + resized_width = normed_height_; + } + + unsigned char* resized_pixels = nullptr; + if (!need_resize) { + resized_pixels = const_cast(pixels); + } else { + // Bilinear Interpolation Resize + resized_pixels = static_cast(malloc( + resized_height * resized_width * channel * sizeof(unsigned char))); + image::utils::resize_hwc(pixels, + resized_pixels, + height, + width, + channel, + resized_height, + resized_width); + } + + unsigned char* rotated_pixels = nullptr; + if (config.option == image::NO_ROTATE) { + rotated_pixels = resized_pixels; + } else { + rotated_pixels = static_cast(malloc( + normed_height_ * normed_width_ * channel * sizeof(unsigned char))); + image::utils::rotate_hwc(resized_pixels, + rotated_pixels, + resized_height, + resized_width, + channel, + config.option); + } + + if (true) { + // HWC -> CHW + size_t index = 0; + if (config.format == image::kRGB) { + // RGB/RGBA -> RGB + for (size_t c = 0; c < normed_channel_; ++c) { + for (size_t h = 0; h < normed_height_; ++h) { + for (size_t w = 0; w < normed_width_; ++w) { + normed_pixels[index] = + static_cast( + rotated_pixels[(h * normed_width_ + w) * channel + c]) - + means_[c]; + index++; + } + } + } + } else if (config.format == image::kBGR) { + // BGR/BGRA -> RGB + for (size_t c = 0; c < normed_channel_; ++c) { + for (size_t h = 0; h < normed_height_; ++h) { + for (size_t w = 0; w < normed_width_; ++w) { + normed_pixels[index] = + static_cast( + rotated_pixels[(h * normed_width_ + w) * channel + + (normed_channel_ - 1 - c)]) - + means_[c]; + index++; + } + } + } + } + } + + if (rotated_pixels != nullptr && rotated_pixels != resized_pixels) { + free(rotated_pixels); + rotated_pixels = nullptr; + } + if (resized_pixels != nullptr && resized_pixels != pixels) { + free(resized_pixels); + resized_pixels = nullptr; + } +} + +void ImageRecognizer::infer(const unsigned char* pixels, + const size_t height, + const size_t width, + const size_t channel, + const image::Config& config, + Result& result) { + if (height < normed_height_ || width < normed_width_) { + fprintf(stderr, + "Image size should be no less than the normed size (%u vs %u, %u " + "vs %u).\n", + height, + normed_height_, + width, + normed_width_); + return; + } + + // Step 3: Prepare input Arguments + paddle_arguments in_args = paddle_arguments_create_none(); + + // There is only one input of this network. + CHECK(paddle_arguments_resize(in_args, 1)); + + // Create input matrix. + // Set the value + paddle_matrix mat = paddle_matrix_create( + /* sample_num */ 1, + /* size */ normed_channel_ * normed_height_ * normed_width_, + /* useGPU */ false); + CHECK(paddle_arguments_set_value(in_args, 0, mat)); + + // Get First row. + paddle_real* array; + CHECK(paddle_matrix_get_row(mat, 0, &array)); + + preprocess(pixels, array, height, width, channel, config); + + // Step 4: Do inference. + paddle_arguments out_args = paddle_arguments_create_none(); + { + CHECK(paddle_gradient_machine_forward(gradient_machine_, + in_args, + out_args, + /* isTrain */ false)); + } + + // Step 5: Get the result. + paddle_matrix probs = paddle_matrix_create_none(); + CHECK(paddle_arguments_get_value(out_args, 0, probs)); + + paddle_error err = paddle_matrix_get_row(probs, 0, &result.data); + if (err == kPD_NO_ERROR) { + CHECK(paddle_matrix_get_shape(probs, &result.height, &result.width)); + } + + // Step 6: Release the resources. + CHECK(paddle_arguments_destroy(in_args)); + CHECK(paddle_matrix_destroy(mat)); + CHECK(paddle_arguments_destroy(out_args)); + CHECK(paddle_matrix_destroy(probs)); +} + +void ImageRecognizer::release() { + if (gradient_machine_ != nullptr) { + CHECK(paddle_gradient_machine_destroy(gradient_machine_)); + } +} diff --git a/Demo/iOS/AICamera/AICamera/paddle_image_recognizer.h b/Demo/iOS/AICamera/AICamera/paddle_image_recognizer.h new file mode 100644 index 0000000..91e02d9 --- /dev/null +++ b/Demo/iOS/AICamera/AICamera/paddle_image_recognizer.h @@ -0,0 +1,102 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License */ + +#pragma once + +#include +#include +#include +#include +#include "image_utils.h" + +static const char* paddle_error_string(paddle_error status) { + switch (status) { + case kPD_NULLPTR: + return "nullptr error"; + case kPD_OUT_OF_RANGE: + return "out of range error"; + case kPD_PROTOBUF_ERROR: + return "protobuf error"; + case kPD_NOT_SUPPORTED: + return "not supported error"; + case kPD_UNDEFINED_ERROR: + return "undefined error"; + default: + return ""; + }; +} + +#define CHECK(stmt) \ + do { \ + paddle_error __err__ = stmt; \ + if (__err__ != kPD_NO_ERROR) { \ + const char* str = paddle_error_string(__err__); \ + fprintf(stderr, "%s (%d) in " #stmt "\n", str, __err__); \ + exit(__err__); \ + } \ + } while (0) + +class ImageRecognizer { +public: + struct Result { + Result() : data(nullptr), height(0), width(0) {} + + float* data; + uint64_t height; + uint64_t width; + }; + +public: + ImageRecognizer() + : gradient_machine_(nullptr), + normed_height_(0), + normed_width_(0), + normed_channel_(0) {} + + static void init_paddle() { + // Initalize Paddle + char* argv[] = {const_cast("--use_gpu=False"), + const_cast("--pool_limit_size=0")}; + CHECK(paddle_init(2, (char**)argv)); + } + + void init(const char* merged_model_path, + const size_t normed_height, + const size_t normed_width, + const size_t normed_channel, + const std::vector& means); + void infer(const unsigned char* pixels, + const size_t height, + const size_t width, + const size_t channel, + const image::Config& config, + Result& result); + void release(); + +protected: + void preprocess(const unsigned char* pixels, + float* normed_pixels, + const size_t height, + const size_t width, + const size_t channel, + const image::Config& config); + +private: + paddle_gradient_machine gradient_machine_; + + size_t normed_height_; + size_t normed_width_; + size_t normed_channel_; + std::vector means_; +}; diff --git a/Demo/iOS/AICamera/PDCamera.xcodeproj/project.pbxproj b/Demo/iOS/AICamera/PDCamera.xcodeproj/project.pbxproj new file mode 100644 index 0000000..07a47c1 --- /dev/null +++ b/Demo/iOS/AICamera/PDCamera.xcodeproj/project.pbxproj @@ -0,0 +1,470 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 46; + objects = { + +/* Begin PBXBuildFile section */ + 480C6CF51FB16CD80009876E /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 480C6CF41FB16CD80009876E /* AppDelegate.swift */; }; + 480C6CF71FB16CD80009876E /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 480C6CF61FB16CD80009876E /* ViewController.swift */; }; + 480C6CFA1FB16CD80009876E /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 480C6CF81FB16CD80009876E /* Main.storyboard */; }; + 480C6CFC1FB16CD80009876E /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 480C6CFB1FB16CD80009876E /* Assets.xcassets */; }; + 480C6CFF1FB16CD80009876E /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 480C6CFD1FB16CD80009876E /* LaunchScreen.storyboard */; }; + 480C6D071FB242CA0009876E /* SSDDrawLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 480C6D061FB242CA0009876E /* SSDDrawLayer.swift */; }; + 480C6D091FB262BF0009876E /* ImageRecognizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 480C6D081FB262BF0009876E /* ImageRecognizer.swift */; }; + 483332F11FD8B0D800699E24 /* image_utils.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 483332ED1FD8B0D700699E24 /* image_utils.cpp */; }; + 483332F21FD8B0D800699E24 /* paddle_image_recognizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 483332F01FD8B0D800699E24 /* paddle_image_recognizer.cpp */; }; + 488E78C91FB290580075A8E3 /* Accelerate.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 488E78C81FB290580075A8E3 /* Accelerate.framework */; }; + 488E78CC1FB290AE0075A8E3 /* libpaddle_capi_engine.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 488E78CA1FB290AE0075A8E3 /* libpaddle_capi_engine.a */; }; + 488E78CD1FB290AE0075A8E3 /* libpaddle_capi_layers.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 488E78CB1FB290AE0075A8E3 /* libpaddle_capi_layers.a */; }; + 488E78CF1FB290E40075A8E3 /* libgflags.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 488E78CE1FB290E40075A8E3 /* libgflags.a */; }; + 488E78D11FB290EF0075A8E3 /* libglog.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 488E78D01FB290EF0075A8E3 /* libglog.a */; }; + 488E78D31FB290FA0075A8E3 /* libprotobuf.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 488E78D21FB290FA0075A8E3 /* libprotobuf.a */; }; + 488E78D51FB291030075A8E3 /* libz.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 488E78D41FB291030075A8E3 /* libz.a */; }; + 48C36BFB1FB8320C009CDE05 /* SSDData.m in Sources */ = {isa = PBXBuildFile; fileRef = 48C36BFA1FB8320C009CDE05 /* SSDData.m */; }; + 48C36C001FBA529C009CDE05 /* PaddleHelper.mm in Sources */ = {isa = PBXBuildFile; fileRef = 48C36BFF1FBA529B009CDE05 /* PaddleHelper.mm */; }; + 48C36C051FBAD309009CDE05 /* pascal_mobilenet_300_66.paddle in Resources */ = {isa = PBXBuildFile; fileRef = 48C36C041FBAC702009CDE05 /* pascal_mobilenet_300_66.paddle */; }; + 48C36C081FBBA721009CDE05 /* SSDModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48C36C071FBBA721009CDE05 /* SSDModel.swift */; }; + 48C36C0B1FBC18D5009CDE05 /* face_mobilenet_160_91.paddle in Resources */ = {isa = PBXBuildFile; fileRef = 48C36C0A1FBC18C6009CDE05 /* face_mobilenet_160_91.paddle */; }; + 48CAD1441FB2B3B3009A0523 /* ImageRecognizerPaddleWrapper.mm in Sources */ = {isa = PBXBuildFile; fileRef = 48CAD1431FB2B3B3009A0523 /* ImageRecognizerPaddleWrapper.mm */; }; + D09B082B1FB280CD0014EF43 /* SSDMultiboxLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09B082A1FB280CD0014EF43 /* SSDMultiboxLayer.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 480C6CF11FB16CD80009876E /* PDCamera.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = PDCamera.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 480C6CF41FB16CD80009876E /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; + 480C6CF61FB16CD80009876E /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; + 480C6CF91FB16CD80009876E /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; + 480C6CFB1FB16CD80009876E /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + 480C6CFE1FB16CD80009876E /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; + 480C6D001FB16CD80009876E /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 480C6D061FB242CA0009876E /* SSDDrawLayer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SSDDrawLayer.swift; sourceTree = ""; }; + 480C6D081FB262BF0009876E /* ImageRecognizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ImageRecognizer.swift; sourceTree = ""; }; + 483332ED1FD8B0D700699E24 /* image_utils.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = image_utils.cpp; sourceTree = ""; }; + 483332EE1FD8B0D700699E24 /* image_utils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = image_utils.h; sourceTree = ""; }; + 483332EF1FD8B0D800699E24 /* paddle_image_recognizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = paddle_image_recognizer.h; sourceTree = ""; }; + 483332F01FD8B0D800699E24 /* paddle_image_recognizer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = paddle_image_recognizer.cpp; sourceTree = ""; }; + 483332F31FD8B13A00699E24 /* image.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = image.h; sourceTree = ""; }; + 4883B2371FB4E7FE00F9204A /* ImageRecognizerPaddleWrapper.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ImageRecognizerPaddleWrapper.h; sourceTree = ""; }; + 488E78C81FB290580075A8E3 /* Accelerate.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Accelerate.framework; path = System/Library/Frameworks/Accelerate.framework; sourceTree = SDKROOT; }; + 488E78CA1FB290AE0075A8E3 /* libpaddle_capi_engine.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libpaddle_capi_engine.a; path = "paddle-ios/lib/libpaddle_capi_engine.a"; sourceTree = ""; }; + 488E78CB1FB290AE0075A8E3 /* libpaddle_capi_layers.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libpaddle_capi_layers.a; path = "paddle-ios/lib/libpaddle_capi_layers.a"; sourceTree = ""; }; + 488E78CE1FB290E40075A8E3 /* libgflags.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libgflags.a; path = "paddle-ios/third_party/gflags/lib/libgflags.a"; sourceTree = ""; }; + 488E78D01FB290EF0075A8E3 /* libglog.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libglog.a; path = "paddle-ios/third_party/glog/lib/libglog.a"; sourceTree = ""; }; + 488E78D21FB290FA0075A8E3 /* libprotobuf.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libprotobuf.a; path = "paddle-ios/third_party/protobuf/lib/libprotobuf.a"; sourceTree = ""; }; + 488E78D41FB291030075A8E3 /* libz.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libz.a; path = "paddle-ios/third_party/zlib/lib/libz.a"; sourceTree = ""; }; + 48C36BFA1FB8320C009CDE05 /* SSDData.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SSDData.m; sourceTree = ""; }; + 48C36BFE1FB83241009CDE05 /* SSDData.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SSDData.h; sourceTree = ""; }; + 48C36BFF1FBA529B009CDE05 /* PaddleHelper.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = PaddleHelper.mm; sourceTree = ""; }; + 48C36C011FBA544C009CDE05 /* PaddleHelper.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PaddleHelper.h; sourceTree = ""; }; + 48C36C041FBAC702009CDE05 /* pascal_mobilenet_300_66.paddle */ = {isa = PBXFileReference; lastKnownFileType = file; path = pascal_mobilenet_300_66.paddle; sourceTree = ""; }; + 48C36C071FBBA721009CDE05 /* SSDModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSDModel.swift; sourceTree = ""; }; + 48C36C0A1FBC18C6009CDE05 /* face_mobilenet_160_91.paddle */ = {isa = PBXFileReference; lastKnownFileType = file; path = face_mobilenet_160_91.paddle; sourceTree = ""; }; + 48CAD13E1FB29318009A0523 /* AICamera-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "AICamera-Bridging-Header.h"; sourceTree = ""; }; + 48CAD1431FB2B3B3009A0523 /* ImageRecognizerPaddleWrapper.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = ImageRecognizerPaddleWrapper.mm; sourceTree = ""; }; + D09B082A1FB280CD0014EF43 /* SSDMultiboxLayer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSDMultiboxLayer.swift; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 480C6CEE1FB16CD80009876E /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 488E78CC1FB290AE0075A8E3 /* libpaddle_capi_engine.a in Frameworks */, + 488E78D11FB290EF0075A8E3 /* libglog.a in Frameworks */, + 488E78CF1FB290E40075A8E3 /* libgflags.a in Frameworks */, + 488E78CD1FB290AE0075A8E3 /* libpaddle_capi_layers.a in Frameworks */, + 488E78D31FB290FA0075A8E3 /* libprotobuf.a in Frameworks */, + 488E78D51FB291030075A8E3 /* libz.a in Frameworks */, + 488E78C91FB290580075A8E3 /* Accelerate.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 480520A71FB3DA18008482C5 /* models */ = { + isa = PBXGroup; + children = ( + 48C36C0A1FBC18C6009CDE05 /* face_mobilenet_160_91.paddle */, + 48C36C041FBAC702009CDE05 /* pascal_mobilenet_300_66.paddle */, + ); + path = models; + sourceTree = ""; + }; + 480C6CE81FB16CD80009876E = { + isa = PBXGroup; + children = ( + 480520A71FB3DA18008482C5 /* models */, + 480C6CF31FB16CD80009876E /* AICamera */, + 480C6CF21FB16CD80009876E /* Products */, + 488E78C71FB2901C0075A8E3 /* Frameworks */, + ); + sourceTree = ""; + }; + 480C6CF21FB16CD80009876E /* Products */ = { + isa = PBXGroup; + children = ( + 480C6CF11FB16CD80009876E /* PDCamera.app */, + ); + name = Products; + sourceTree = ""; + }; + 480C6CF31FB16CD80009876E /* AICamera */ = { + isa = PBXGroup; + children = ( + 480C6CF41FB16CD80009876E /* AppDelegate.swift */, + 480C6D081FB262BF0009876E /* ImageRecognizer.swift */, + 480C6CF61FB16CD80009876E /* ViewController.swift */, + 480C6D061FB242CA0009876E /* SSDDrawLayer.swift */, + D09B082A1FB280CD0014EF43 /* SSDMultiboxLayer.swift */, + 483332F31FD8B13A00699E24 /* image.h */, + 483332ED1FD8B0D700699E24 /* image_utils.cpp */, + 483332EE1FD8B0D700699E24 /* image_utils.h */, + 483332F01FD8B0D800699E24 /* paddle_image_recognizer.cpp */, + 483332EF1FD8B0D800699E24 /* paddle_image_recognizer.h */, + 48C36BFF1FBA529B009CDE05 /* PaddleHelper.mm */, + 48C36C011FBA544C009CDE05 /* PaddleHelper.h */, + 48CAD1431FB2B3B3009A0523 /* ImageRecognizerPaddleWrapper.mm */, + 4883B2371FB4E7FE00F9204A /* ImageRecognizerPaddleWrapper.h */, + 48C36C071FBBA721009CDE05 /* SSDModel.swift */, + 48C36BFA1FB8320C009CDE05 /* SSDData.m */, + 48C36BFE1FB83241009CDE05 /* SSDData.h */, + 480C6CF81FB16CD80009876E /* Main.storyboard */, + 480C6CFB1FB16CD80009876E /* Assets.xcassets */, + 480C6CFD1FB16CD80009876E /* LaunchScreen.storyboard */, + 480C6D001FB16CD80009876E /* Info.plist */, + 48CAD13E1FB29318009A0523 /* AICamera-Bridging-Header.h */, + ); + path = AICamera; + sourceTree = ""; + }; + 488E78C71FB2901C0075A8E3 /* Frameworks */ = { + isa = PBXGroup; + children = ( + 488E78D41FB291030075A8E3 /* libz.a */, + 488E78D21FB290FA0075A8E3 /* libprotobuf.a */, + 488E78D01FB290EF0075A8E3 /* libglog.a */, + 488E78CE1FB290E40075A8E3 /* libgflags.a */, + 488E78CA1FB290AE0075A8E3 /* libpaddle_capi_engine.a */, + 488E78CB1FB290AE0075A8E3 /* libpaddle_capi_layers.a */, + 488E78C81FB290580075A8E3 /* Accelerate.framework */, + ); + name = Frameworks; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 480C6CF01FB16CD80009876E /* PDCamera */ = { + isa = PBXNativeTarget; + buildConfigurationList = 480C6D031FB16CD80009876E /* Build configuration list for PBXNativeTarget "PDCamera" */; + buildPhases = ( + 480C6CED1FB16CD80009876E /* Sources */, + 480C6CEE1FB16CD80009876E /* Frameworks */, + 480C6CEF1FB16CD80009876E /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = PDCamera; + productName = AICamera; + productReference = 480C6CF11FB16CD80009876E /* PDCamera.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 480C6CE91FB16CD80009876E /* Project object */ = { + isa = PBXProject; + attributes = { + LastSwiftUpdateCheck = 0830; + LastUpgradeCheck = 0910; + ORGANIZATIONNAME = PaddlePaddle; + TargetAttributes = { + 480C6CF01FB16CD80009876E = { + CreatedOnToolsVersion = 8.3.3; + DevelopmentTeam = H8UUXRDF6H; + LastSwiftMigration = 0910; + ProvisioningStyle = Automatic; + }; + }; + }; + buildConfigurationList = 480C6CEC1FB16CD80009876E /* Build configuration list for PBXProject "PDCamera" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = English; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 480C6CE81FB16CD80009876E; + productRefGroup = 480C6CF21FB16CD80009876E /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 480C6CF01FB16CD80009876E /* PDCamera */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 480C6CEF1FB16CD80009876E /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 48C36C0B1FBC18D5009CDE05 /* face_mobilenet_160_91.paddle in Resources */, + 48C36C051FBAD309009CDE05 /* pascal_mobilenet_300_66.paddle in Resources */, + 480C6CFF1FB16CD80009876E /* LaunchScreen.storyboard in Resources */, + 480C6CFC1FB16CD80009876E /* Assets.xcassets in Resources */, + 480C6CFA1FB16CD80009876E /* Main.storyboard in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 480C6CED1FB16CD80009876E /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 480C6CF71FB16CD80009876E /* ViewController.swift in Sources */, + 480C6CF51FB16CD80009876E /* AppDelegate.swift in Sources */, + 483332F21FD8B0D800699E24 /* paddle_image_recognizer.cpp in Sources */, + D09B082B1FB280CD0014EF43 /* SSDMultiboxLayer.swift in Sources */, + 48CAD1441FB2B3B3009A0523 /* ImageRecognizerPaddleWrapper.mm in Sources */, + 48C36C081FBBA721009CDE05 /* SSDModel.swift in Sources */, + 483332F11FD8B0D800699E24 /* image_utils.cpp in Sources */, + 480C6D071FB242CA0009876E /* SSDDrawLayer.swift in Sources */, + 48C36C001FBA529C009CDE05 /* PaddleHelper.mm in Sources */, + 480C6D091FB262BF0009876E /* ImageRecognizer.swift in Sources */, + 48C36BFB1FB8320C009CDE05 /* SSDData.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXVariantGroup section */ + 480C6CF81FB16CD80009876E /* Main.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 480C6CF91FB16CD80009876E /* Base */, + ); + name = Main.storyboard; + sourceTree = ""; + }; + 480C6CFD1FB16CD80009876E /* LaunchScreen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 480C6CFE1FB16CD80009876E /* Base */, + ); + name = LaunchScreen.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 480C6D011FB16CD80009876E /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + DEFINES_MODULE = YES; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = "${PROJECT_DIR}/paddle-ios/include"; + IPHONEOS_DEPLOYMENT_TARGET = 10.3; + MTL_ENABLE_DEBUG_INFO = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 480C6D021FB16CD80009876E /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + DEFINES_MODULE = YES; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = "${PROJECT_DIR}/paddle-ios/include"; + IPHONEOS_DEPLOYMENT_TARGET = 10.3; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 480C6D041FB16CD80009876E /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ENABLE_MODULES = YES; + DEVELOPMENT_TEAM = H8UUXRDF6H; + HEADER_SEARCH_PATHS = "${PROJECT_DIR}/paddle-ios/include/**"; + INFOPLIST_FILE = AICamera/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 10.0; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)/paddle-ios/lib", + "$(PROJECT_DIR)/paddle-ios/third_party/gflags/lib", + "$(PROJECT_DIR)/paddle-ios/third_party/glog/lib", + "$(PROJECT_DIR)/paddle-ios/third_party/protobuf/lib", + "$(PROJECT_DIR)/paddle-ios/third_party/zlib/lib", + ); + ONLY_ACTIVE_ARCH = NO; + OTHER_LDFLAGS = ( + "-force_load", + "${PROJECT_DIR}/paddle-ios/lib/libpaddle_capi_layers.a", + ); + PRODUCT_BUNDLE_IDENTIFIER = com.paddlepaddle.PDCamera; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "AICamera/AICamera-Bridging-Header.h"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_SWIFT3_OBJC_INFERENCE = On; + SWIFT_VERSION = 3.0; + VALID_ARCHS = arm64; + }; + name = Debug; + }; + 480C6D051FB16CD80009876E /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ENABLE_MODULES = YES; + DEVELOPMENT_TEAM = H8UUXRDF6H; + HEADER_SEARCH_PATHS = "${PROJECT_DIR}/paddle-ios/include/**"; + INFOPLIST_FILE = AICamera/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 10.0; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)/paddle-ios/lib", + "$(PROJECT_DIR)/paddle-ios/third_party/gflags/lib", + "$(PROJECT_DIR)/paddle-ios/third_party/glog/lib", + "$(PROJECT_DIR)/paddle-ios/third_party/protobuf/lib", + "$(PROJECT_DIR)/paddle-ios/third_party/zlib/lib", + ); + ONLY_ACTIVE_ARCH = NO; + OTHER_LDFLAGS = ( + "-force_load", + "${PROJECT_DIR}/paddle-ios/lib/libpaddle_capi_layers.a", + ); + PRODUCT_BUNDLE_IDENTIFIER = com.paddlepaddle.PDCamera; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "AICamera/AICamera-Bridging-Header.h"; + SWIFT_SWIFT3_OBJC_INFERENCE = On; + SWIFT_VERSION = 3.0; + VALID_ARCHS = arm64; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 480C6CEC1FB16CD80009876E /* Build configuration list for PBXProject "PDCamera" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 480C6D011FB16CD80009876E /* Debug */, + 480C6D021FB16CD80009876E /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 480C6D031FB16CD80009876E /* Build configuration list for PBXNativeTarget "PDCamera" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 480C6D041FB16CD80009876E /* Debug */, + 480C6D051FB16CD80009876E /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 480C6CE91FB16CD80009876E /* Project object */; +} diff --git a/Demo/iOS/AICamera/PDCamera.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/Demo/iOS/AICamera/PDCamera.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..27a9d67 --- /dev/null +++ b/Demo/iOS/AICamera/PDCamera.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/Demo/iOS/AICamera/PDCamera.xcodeproj/project.xcworkspace/xcuserdata/Nickychan.xcuserdatad/UserInterfaceState.xcuserstate b/Demo/iOS/AICamera/PDCamera.xcodeproj/project.xcworkspace/xcuserdata/Nickychan.xcuserdatad/UserInterfaceState.xcuserstate new file mode 100644 index 0000000..3b1c122 Binary files /dev/null and b/Demo/iOS/AICamera/PDCamera.xcodeproj/project.xcworkspace/xcuserdata/Nickychan.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist b/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist new file mode 100644 index 0000000..a643924 --- /dev/null +++ b/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist @@ -0,0 +1,135 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcschemes/AICamera.xcscheme b/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcschemes/AICamera.xcscheme new file mode 100644 index 0000000..8ddd085 --- /dev/null +++ b/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcschemes/AICamera.xcscheme @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcschemes/xcschememanagement.plist b/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000..50e0adf --- /dev/null +++ b/Demo/iOS/AICamera/PDCamera.xcodeproj/xcuserdata/Nickychan.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,24 @@ + + + + + SchemeUserState + + AICamera.xcscheme + + isShown + + orderHint + 1 + + + SuppressBuildableAutocreation + + 480C6CF01FB16CD80009876E + + primary + + + + + diff --git a/Demo/iOS/AICamera/README.md b/Demo/iOS/AICamera/README.md new file mode 100644 index 0000000..01a1468 --- /dev/null +++ b/Demo/iOS/AICamera/README.md @@ -0,0 +1,103 @@ +# PDCamera iOS Demo with SSD Model + +This iOS demo shows PaddlePaddle running SSD(Single Shot MultiBox Detector）Object detection on iOS devices locally and offline. It loads a pretrained model with PaddlePaddle and uses camera to capture images and call PaddlePaddle's inference ability to show detected objects to users. + +You can look at SSD model architecture [here](https://github.com/PaddlePaddle/models/tree/develop/ssd) and a linux demo [here](https://github.com/PaddlePaddle/Mobile/tree/develop/Demo/linux) + + +## Download and run the app + +To simply run the demo with iPhone/iPad, scan the QR code below, click "Install PDCamera" in the link and the app will be downloaded in the background. +After installed, go to Settings -> General -> Device Management -> Baidu USA llc -> Trust "Baidu USA llc" + + +### QR code link + +[image](https://github.com/PaddlePaddle/Mobile/tree/develop/Demo/iOS/AICamera/assets/qr_code_ios.png) + +### Demo screenshot + +[image](https://github.com/PaddlePaddle/Mobile/tree/develop/Demo/iOS/AICamera/assets/demo_screenshot.jpg) + +Detected object will be highlighted as a bounding box with a classified object label and probability. + + +## Classifications +pascal_mobilenet_300_66 and vgg_ssd_net models can only classify following 20 objects: + +- aeroplane +- bicycle +- background +- boat +- bottle +- bus +- car +- cat +- chair +- cow +- diningtable +- dog +- horse +- motorbike +- person +- pottedplant +- sheep +- sofa +- train +- tvmonitor + +face_mobilenet_160_91 can only classify human's face + + +## Settings + +Simply tap on the screen to toggle settings + +- Models: Select Pascal MobileNet 300 or Face MobileNet 160, App will exit, need to launch to restart. +- Camera: Toggle Front/Back Camera. App will exit, need to launch to restart. +- Accuracy Threshold: Adjust threshold to filter more/less objects based on probability +- Time Refresh Rate: Adjust the time to refresh bounding box more/less frequently + + +## Development or modify + +Use latest XCode for development. This demo requires a camera for object detection, therefore you must use a device (iPhone or iPad) for development and testing. Simulators will not work as they cannot access camera. + +For developers, feel free to use this as a reference to start a new project. This demo fully demonstrates how to integrate Paddle C Library to iOS and called from Swift. + +Swift cannot directly call C API, in order to have client in Swift work, create Objective-C briding header and a Objective-C++ wrapper (.mm files) to access paddle APIs. + + +## Integrate Paddle C Library to iOS + +-Follow this guide [Build PaddlePaddle for iOS](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/mobile/cross_compiling_for_ios_cn.md) to generate paddle libs(include, lib, third_party). +-Create a folder paddle-ios and add to project root. Put the 3 paddle libs folder under paddle-ios. +- Add the `include` directory to **Header Search Paths** +![image](https://user-images.githubusercontent.com/12538138/32491809-b215cf7a-c37d-11e7-87f8-3d45f07bc63e.png) + +- Add the `Accelerate.framework` or `veclib.framework` to your project, if your PaddlePaddle is built with `IOS_USE_VECLIB_FOR_BLAS=ON` +- Add the libraries of paddle, `libpaddle_capi_layers.a` and `libpaddle_capi_engine.a`, and all the third party libraries to your project + + +- Set `-force_load` for `libpaddle_capi_layers.a` +![image](https://user-images.githubusercontent.com/12538138/32492328-8504ebae-c37f-11e7-98b5-41615519fbb3.png) + + +## Download Models + +Our models are too large to upload to Github. Create a model folder and add to project root. Download [face_mobilenet_160_91.paddle](http://cloud.dlnel.org/filepub/?uuid=038c1dbf-08b3-42a9-b2dc-efccd63859fb) and [pascal_mobilenet_300_66.paddle](http://cloud.dlnel.org/filepub/?uuid=39c325d9-b468-4940-ba47-d50c8ec5fd5b) to the model folder. + +(Optional) VGG model is relatively large and takes much higher memory(~800Mb), power, and much slower (~1.5secs) on each inference but it has slightly accuracy gain (See below section) +Note: Only runs on iPhone6s or above (iPhone 6 or below will crash due to memory limit) +If you want to try it out, download [vgg_ssd_net.paddle](http://cloud.dlnel.org/filepub/?uuid=1116a5f3-7762-44b5-82bb-9954159cb5d4), then go to +XCode target -> Bulid Phases -> Copy Bundle Resources, click '+' to add vgg_ssd_net.paddle + + +## Accuracy + +| Model | Dimensions | Accuracy | +| ------------------------ |:----------:| --------:| +| face_mobilenet_160_91 | 160x160 | 91% | +| pascal_mobilenet_300_66 | 300x300 | 66% | +| vgg_ssd_net | 300x300 | 71% | + diff --git a/Demo/iOS/AICamera/assets/demo_screenshot.jpg b/Demo/iOS/AICamera/assets/demo_screenshot.jpg new file mode 100644 index 0000000..9aa74cb Binary files /dev/null and b/Demo/iOS/AICamera/assets/demo_screenshot.jpg differ diff --git a/Demo/iOS/AICamera/assets/qr_code_ios.png b/Demo/iOS/AICamera/assets/qr_code_ios.png new file mode 100644 index 0000000..596d241 Binary files /dev/null and b/Demo/iOS/AICamera/assets/qr_code_ios.png differ