-
Notifications
You must be signed in to change notification settings - Fork 3
/
ViewController.m
240 lines (189 loc) · 8.49 KB
/
ViewController.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
//
// ViewController.m
// BlindAssist
//
// Created by Giovanni Terlingen on 27-03-18.
// Copyright © 2018 Giovanni Terlingen. All rights reserved.
//
#import <CoreML/CoreML.h>
#import "cityscapes.h"
#import "ViewController.h"
#include "blindassist.h"
static const NSTimeInterval GravityCheckInterval = 5.0;
/**
* Defines the delay between predictions
*/
static const NSTimeInterval PredictionInterval = 3.0;
BOOL IsFacingHorizon = true;
UInt64 LastPredicitionTime;
@implementation ViewController
- (void)viewDidLoad {
[super viewDidLoad];
[self setTts:[[AVSpeechSynthesizer alloc] init]];
[self speak:@"Initializing application"];
self.motionManager = [[CMMotionManager alloc] init];
self.motionManager.deviceMotionUpdateInterval = GravityCheckInterval;
[self.motionManager startDeviceMotionUpdatesToQueue:[NSOperationQueue currentQueue]
withHandler:^(CMDeviceMotion *motionData, NSError *error) {
[self handleGravity:motionData.gravity];
}];
VNCoreMLModel *model = [VNCoreMLModel modelForMLModel: [[[cityscapes alloc] init] model] error:nil];
[self setRequest:[[VNCoreMLRequest alloc] initWithModel:model completionHandler:^(VNRequest * _Nonnull request, NSError * _Nullable error) {
[self handlePrediction:request :error];
}]];
[self request].imageCropAndScaleOption = VNImageCropAndScaleOptionCenterCrop;
[self speak:@"Finished loading, detecting environment"];
}
-(void)viewDidAppear:(BOOL)animated {
[super viewDidAppear:animated];
[AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo completionHandler:^(BOOL granted) {
if (granted) {
[self permissions:granted];
}
}];
}
-(void)permissions:(BOOL)granted {
if (granted && [self session] == nil) {
[self setupSession];
}
}
-(void)setupSession {
AVCaptureDevice *device = [AVCaptureDevice defaultDeviceWithDeviceType:AVCaptureDeviceTypeBuiltInWideAngleCamera mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionBack];
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:nil];
AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init];
[output setSampleBufferDelegate:self queue:dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0)];
AVCaptureSession *session = [[AVCaptureSession alloc] init];
[session addInput:input];
[session addOutput:output];
AVCaptureVideoPreviewLayer *previewLayer = [AVCaptureVideoPreviewLayer layerWithSession:session];
dispatch_async(dispatch_get_main_queue(), ^{
[[self cameraPreview] addCaptureVideoPreviewLayer:previewLayer];
});
[self setSession:session];
[[self session] startRunning];
}
-(void)deviceOrientationDidChange {
for (AVCaptureOutput *output in [self session].outputs) {
for (AVCaptureConnection *connection in output.connections) {
connection.videoOrientation = [Utils getVideoOrientation];
}
}
}
-(void)handlePrediction:(VNRequest*)request :(NSError*)error {
NSArray *results = [request.results copy];
MLMultiArray *multiArray = ((VNCoreMLFeatureValueObservation*)(results[0])).featureValue.multiArrayValue;
// Shape of MLMultiArray is sequence: channels, height and width
int channels = multiArray.shape[0].intValue;
int height = multiArray.shape[1].intValue;
int width = multiArray.shape[2].intValue;
// Holds the temporary maxima, and its index (only works if less than 256 channels!)
double *tmax = (double*) malloc(width * height * sizeof(double));
uint8_t *tchan = (uint8_t*) malloc(width * height);
double *pointer = (double*) multiArray.dataPointer;
int cStride = multiArray.strides[0].intValue;
int hStride = multiArray.strides[1].intValue;
int wStride = multiArray.strides[2].intValue;
// Just copy the first channel as starting point
for (int h = 0; h < height; h++) {
for (int w = 0; w < width; w++) {
tmax[w + h * width] = pointer[h * hStride + w * wStride];
tchan[w + h * width] = 0; // first channel
}
}
// We skip first channel on purpose.
for (int c = 1; c < channels; c++) {
for (int h = 0; h < height; h++) {
for (int w = 0; w < width; w++) {
double sample = pointer[h * hStride + w * wStride + c * cStride];
if (sample > tmax[w + h * width]) {
tmax[w + h * width] = sample;
tchan[w + h * width] = c;
}
}
}
}
// Now free the maximum buffer, useless
free(tmax);
// Holds the segmented image
uint8_t *bytes = (uint8_t*) malloc(width * height * 4);
// Calculate image color
for (int i = 0; i < height * width; i++) {
struct Color rgba = colors[tchan[i]];
bytes[i * 4 + 0] = (rgba.r);
bytes[i * 4 + 1] = (rgba.g);
bytes[i * 4 + 2] = (rgba.b);
bytes[i * 4 + 3] = (255 / 2); // semi transparent
}
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
CGContextRef context = CGBitmapContextCreate(bytes, width, height, 8, 4 * width, colorSpace, kCGImageAlphaPremultipliedLast);
CFRelease(colorSpace);
CGImageRef cgImage = CGBitmapContextCreateImage(context);
CGContextRelease(context);
UIImage *image = [UIImage imageWithCGImage:cgImage scale:0 orientation:UIImageOrientationUp];
CGImageRelease(cgImage);
dispatch_async(dispatch_get_main_queue(), ^{
[[self predictionView] setImage:image];
});
free(bytes);
UInt64 CurrentTime = [[NSDate date] timeIntervalSince1970];
if (IsFacingHorizon) {
// predict results for this frame
analyse_frame(tchan, height, width);
scene_information information = {};
int result = poll_results(&information);
if (result == SUCCESS) {
// Critical warnings, ignore time delay
if (LastPredicitionTime == 0 || (CurrentTime - LastPredicitionTime) > PredictionInterval) {
if (information.poles_detected > 0) {
[self speak:@"Poles detected."];
}
if (information.vehicle_detected > 0) {
[self speak:@"Parked car detected."];
}
if (information.bikes_detected > 0) {
[self speak:@"Bikes detected."];
}
if (information.walk_position == FRONT) {
[self speak:@"You can walk in front of you."];
} else if (information.walk_position == LEFT) {
[self speak:@"You can walk left."];
} else if (information.walk_position == RIGHT) {
[self speak:@"You can walk right."];
}
LastPredicitionTime = [[NSDate date] timeIntervalSince1970];
}
}
}
// Free t buffer
free(tchan);
}
-(void)speak:(NSString*) string {
AVSpeechUtterance *utterance = [AVSpeechUtterance speechUtteranceWithString:string];
utterance.voice = [AVSpeechSynthesisVoice voiceWithLanguage:@"en-US"];
utterance.rate = AVSpeechUtteranceMaximumSpeechRate * 0.60;
[[self tts] speakUtterance:utterance];
}
- (void)didReceiveMemoryWarning {
[super didReceiveMemoryWarning];
}
- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {
[self deviceOrientationDidChange];
CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (!pixelBuffer) {
return;
}
NSMutableDictionary<VNImageOption, id> *requestOptions = [NSMutableDictionary dictionary];
CFTypeRef cameraIntrinsicData = CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_CameraIntrinsicMatrix, nil);
requestOptions[VNImageOptionCameraIntrinsics] = (__bridge id)(cameraIntrinsicData);
VNImageRequestHandler *handler = [[VNImageRequestHandler alloc] initWithCVPixelBuffer:pixelBuffer options:requestOptions];
[handler performRequests:@[[self request]] error:nil];
}
-(void)handleGravity:(CMAcceleration)gravity
{
IsFacingHorizon = gravity.y <= -0.97f && gravity.y <= 1.0f;
if (!IsFacingHorizon) {
// TODO: Make some beep for this
[self speak:@"Warning: camera is not facing the horizon."];
}
}
@end