Skip to content
This repository has been archived by the owner on Jan 12, 2019. It is now read-only.

Commit

Permalink
expiry debug: collect illustrative images
Browse files Browse the repository at this point in the history
  • Loading branch information
Dave Goldman committed Apr 27, 2015
1 parent 1f7cb22 commit f7920a8
Show file tree
Hide file tree
Showing 3 changed files with 167 additions and 1 deletion.
4 changes: 4 additions & 0 deletions mz_ios.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,8 @@
// from_points should have the following ordering: top-left, top-right, bottom-left, bottom-right
void ios_gpu_unwarp(dmz_context *dmz, IplImage *input, const dmz_point from_points[4], IplImage *output);

#if DMZ_DEBUG
void ios_save_file(char *filename, IplImage *image);
#endif

#endif
16 changes: 16 additions & 0 deletions mz_ios.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,20 @@ void ios_gpu_unwarp(dmz_context *dmz, IplImage *input, const dmz_point from_poin
[filter processIplImage:input dstIplImg:output];
}

#if DMZ_DEBUG
void ios_save_file(char *filename, IplImage *image) {
IplImage *localImage = cvCreateImage(cvGetSize(image), image->depth, image->nChannels);
cvConvertScale(image, localImage);

CardIOIplImage *cardioImage = [CardIOIplImage imageWithIplImage:localImage];
UIImage *uiImage = [cardioImage UIImage];
NSData *imageData = UIImagePNGRepresentation(uiImage);
NSString *filePath = [NSString stringWithCString:filename encoding:NSUTF8StringEncoding];
NSString *applicationDocumentsDir = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES)[0];
NSString *storePath = [applicationDocumentsDir stringByAppendingPathComponent:filePath];

[imageData writeToFile:storePath atomically:YES];
}
#endif

#endif // USE_CAMERA
148 changes: 147 additions & 1 deletion scan/expiry_seg.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,20 @@

//#define DEBUG_EXPIRY_SEGMENTATION_PERFORMANCE 1

//#define DEBUG_EXPIRY_IMAGES 1
#if DEBUG_EXPIRY_IMAGES
static int image_session_count = 0;
static int image_stripe_count = 0;
static char image_filename_string[64];
#endif

// slash categorizer
#include "models/expiry/modelm_730c4cbd.hpp"

#pragma mark - image preparation



DMZ_INTERNAL void prepare_image_for_seg(IplImage *image, IplImage *as_float, CharacterRect *rect) {
// Input image: IPL_DEPTH_8U [0 - 255]
// Data for models: IPL_DEPTH_32F [0.0 - 1.0]
Expand Down Expand Up @@ -333,6 +342,47 @@ DMZ_INTERNAL void optimize_character_rects(IplImage *sobel_image, GroupedRects &
cvResetImageROI(sobel_image);
}

#if DEBUG_EXPIRY_IMAGES
DMZ_INTERNAL void add_rects_to_image(IplImage *image, CharacterRectList &rect_list, int character_width) {
for (CharacterRectListIterator rect = rect_list.begin(); rect != rect_list.end(); ++rect) {
cvRectangleR(image, cvRect(rect->left, rect->top, character_width, kSmallCharacterHeight), cvScalar(SHRT_MAX));
}
}
#endif

#if DEBUG_EXPIRY_IMAGES
DMZ_INTERNAL void save_image_groups(IplImage *image, GroupedRectsList &groups) {
if (!groups.size()) {
return;
}

IplImage *rects_image = cvCreateImage(cvGetSize(image), image->depth, image->nChannels);
cvCopy(image, rects_image);

int min_top = SHRT_MAX;
int max_top = 0;
for (GroupedRectsListIterator group = groups.begin(); group != groups.end(); ++group) {
add_rects_to_image(rects_image, group->character_rects, group->character_width);

cvRectangleR(rects_image, cvRect(group->left - 1, group->top - 1, group->width + 2, group->height + 2), cvScalar(200.0f));

if (group->top < min_top) {
min_top = group->top;
}
if (group->top > max_top) {
max_top = group->top;
}
}

cvSetImageROI(rects_image, cvRect(0,
min_top - kSmallCharacterHeight,
rects_image->width,
MIN(max_top + 2 * kSmallCharacterHeight, image->height) - (min_top - kSmallCharacterHeight)));
ios_save_file(image_filename_string, rects_image);
cvReleaseImage(&rects_image);
}
#endif

DMZ_INTERNAL void find_character_groups_for_stripe(IplImage *card_y, IplImage *sobel_image, int stripe_base_row, long stripe_sum, GroupedRectsList &expiry_groups, GroupedRectsList &name_groups) {
#if DEBUG_EXPIRY_SEGMENTATION_PERFORMANCE
dmz_debug_timer_start(1);
Expand Down Expand Up @@ -434,6 +484,7 @@ DMZ_INTERNAL void find_character_groups_for_stripe(IplImage *card_y, IplImage *s
grouped_rect.character_width = kSmallCharacterWidth;
non_overlapping_rect_list.push_back(grouped_rect);

assert(8 == kSmallCharacterWidth - 1);
non_overlapping_rect_mask[rect->left + 0] = true;
non_overlapping_rect_mask[rect->left + 1] = true;
non_overlapping_rect_mask[rect->left + 2] = true;
Expand All @@ -443,7 +494,6 @@ DMZ_INTERNAL void find_character_groups_for_stripe(IplImage *card_y, IplImage *s
non_overlapping_rect_mask[rect->left + 6] = true;
non_overlapping_rect_mask[rect->left + 7] = true;
non_overlapping_rect_mask[rect->left + 8] = true;
assert(8 == kSmallCharacterWidth - 1);
}
}

Expand All @@ -453,6 +503,28 @@ DMZ_INTERNAL void find_character_groups_for_stripe(IplImage *card_y, IplImage *s
dmz_debug_timer_print(msg2, 1);
#endif

#if DEBUG_EXPIRY_IMAGES
IplImage *rects_image = cvCreateImage(cvGetSize(card_y), card_y->depth, card_y->nChannels);
cvCopy(card_y, rects_image);

int min_top = SHRT_MAX;
CharacterRectList rects;
for (GroupedRectsListIterator group = non_overlapping_rect_list.begin(); group != non_overlapping_rect_list.end(); ++group) {
CharacterRect rect(group->top, group->left, group->sum);
rects.push_back(rect);
if (group->top < min_top) {
min_top = group->top;
}
}

add_rects_to_image(rects_image, rects, kSmallCharacterWidth);
image_stripe_count++;
sprintf(image_filename_string, "%d-e-%d-char_rects.png", image_session_count, image_stripe_count);
cvSetImageROI(rects_image, cvRect(0, min_top - kSmallCharacterHeight, rects_image->width, kSmallCharacterHeight * 3));
ios_save_file(image_filename_string, rects_image);
cvReleaseImage(&rects_image);
#endif

// "local group" = a set of character rects with inter-rect horizontal gaps of less than kSmallCharacterWidth
// "super-group" = a set of local groups with inter-group horizontal gaps of less than 2 * kSmallCharacterWidth
//
Expand Down Expand Up @@ -481,6 +553,11 @@ DMZ_INTERNAL void find_character_groups_for_stripe(IplImage *card_y, IplImage *s
dmz_debug_timer_print(msg4, 1);
#endif

#if DEBUG_EXPIRY_IMAGES
sprintf(image_filename_string, "%d-f-%d-groups.png", image_session_count, image_stripe_count);
save_image_groups(card_y, local_groups);
#endif

// Note: The two loops below use `kMinimumExpiryStripCharacters - 1` rather than `kMinimumExpiryStripCharacters`,
// as you might have expected.
// Sometimes the steps up to this point have gotten slightly confused by a card image, and have misidentified,
Expand Down Expand Up @@ -521,6 +598,11 @@ DMZ_INTERNAL void find_character_groups_for_stripe(IplImage *card_y, IplImage *s
dmz_debug_timer_print("regrid the groups", 1);
#endif

#if DEBUG_EXPIRY_IMAGES
sprintf(image_filename_string, "%d-g-%d-regrid.png", image_session_count, image_stripe_count);
save_image_groups(card_y, local_groups);
#endif

for (int index = (int)local_groups.size() - 1; index >= 0; index--) {
optimize_character_rects(sobel_image, local_groups[index]);
if (local_groups[index].character_rects.size() == 0) {
Expand All @@ -540,6 +622,11 @@ DMZ_INTERNAL void find_character_groups_for_stripe(IplImage *card_y, IplImage *s
dmz_debug_timer_print("shrink the character rects", 1);
#endif

#if DEBUG_EXPIRY_IMAGES
sprintf(image_filename_string, "%d-h-%d-optimize.png", image_session_count, image_stripe_count);
save_image_groups(card_y, local_groups);
#endif

new_groups.clear();
for (GroupedRectsListIterator group = local_groups.begin(); group != local_groups.end(); ++group) {
if (group->character_rects.size() >= kMinimumExpiryStripCharacters) {
Expand Down Expand Up @@ -603,6 +690,11 @@ DMZ_INTERNAL void find_character_groups_for_stripe(IplImage *card_y, IplImage *s
dmz_debug_timer_print("insert local groups into expiry_groups param", 1);
#endif

#if DEBUG_EXPIRY_IMAGES
sprintf(image_filename_string, "%d-i-%d-slash.png", image_session_count, image_stripe_count);
save_image_groups(card_y, expiry_groups);
#endif

// Add supergroups to the passed-in name_groups GroupedRectsList
name_groups.insert(name_groups.end(), super_groups.begin(), super_groups.end());

Expand Down Expand Up @@ -637,6 +729,14 @@ DMZ_INTERNAL void best_expiry_seg(IplImage *card_y, uint16_t starting_y_offset,
dmz_debug_timer_print("do Sobel [Scharr]");
#endif

#if DEBUG_EXPIRY_IMAGES
image_session_count++;
sprintf(image_filename_string, "%d-a-original.png", image_session_count);
ios_save_file(image_filename_string, card_y);
sprintf(image_filename_string, "%d-b-sobel.png", image_session_count);
ios_save_file(image_filename_string, sobel_image);
#endif

cvResetImageROI(card_y);
cvResetImageROI(sobel_image);

Expand All @@ -655,6 +755,33 @@ DMZ_INTERNAL void best_expiry_seg(IplImage *card_y, uint16_t starting_y_offset,
}

cvResetImageROI(sobel_image);

#if DEBUG_EXPIRY_IMAGES
long max_line_sum = 0;
long min_line_sum = LONG_MAX;
for (int row = below_numbers_rect.y; row < sobel_image->height; row++) {
max_line_sum = MAX(line_sum[row], max_line_sum);
min_line_sum = MIN(line_sum[row], min_line_sum);
}
long range = (max_line_sum - min_line_sum);
float scale_factor = range / 255.0f;
int two_thirds_width = (card_image_size.width * 2) / 3;
int one_thirds_width = card_image_size.width - two_thirds_width;

IplImage *rows_image = cvCreateImage(cvGetSize(sobel_image), sobel_image->depth, sobel_image->nChannels);
cvSetZero(rows_image);
cvSetImageROI(sobel_image, cvRect(0, 0, two_thirds_width, sobel_image->height));
cvSetImageROI(rows_image, cvRect(0, 0, two_thirds_width, sobel_image->height));
cvCopy(sobel_image, rows_image);
cvResetImageROI(sobel_image);
for (int row = below_numbers_rect.y; row < sobel_image->height; row++) {
cvSetImageROI(rows_image, cvRect(two_thirds_width, row, one_thirds_width, 1));
cvSet(rows_image, cvScalar((line_sum[row] - min_line_sum) / scale_factor));
}
cvSetImageROI(rows_image, below_numbers_rect);
sprintf(image_filename_string, "%d-c-rows.png", image_session_count);
ios_save_file(image_filename_string, rows_image);
#endif

#if DEBUG_EXPIRY_SEGMENTATION_PERFORMANCE
dmz_debug_timer_print("line sums");
Expand Down Expand Up @@ -741,6 +868,25 @@ DMZ_INTERNAL void best_expiry_seg(IplImage *card_y, uint16_t starting_y_offset,
dmz_debug_timer_print("pick probable stripes");
#endif

#if DEBUG_EXPIRY_IMAGES
int indent = two_thirds_width;
for (std::vector<StripeSum>::iterator probable_stripe = probable_stripes.begin(); probable_stripe != probable_stripes.end(); ++probable_stripe) {
cvSetImageROI(rows_image, cvRect(0, probable_stripe->base_row, two_thirds_width, 1));
cvSet(rows_image, cvScalar(SHRT_MAX));
cvSetImageROI(rows_image, cvRect(0, probable_stripe->base_row + kSmallCharacterHeight - 1, two_thirds_width, 1));
cvSet(rows_image, cvScalar(SHRT_MAX));
cvSetImageROI(rows_image, cvRect(indent, probable_stripe->base_row, 20, kSmallCharacterHeight));
cvSet(rows_image, cvScalar(SHRT_MAX));
indent += 20;
}
cvSetImageROI(rows_image, below_numbers_rect);
sprintf(image_filename_string, "%d-d-stripes.png", image_session_count);
ios_save_file(image_filename_string, rows_image);
cvReleaseImage(&rows_image);

image_stripe_count = 0;
#endif

// For each stripe, find the potential expiry groups and name groups:

for (std::vector<StripeSum>::iterator probable_stripe = probable_stripes.begin(); probable_stripe != probable_stripes.end(); ++probable_stripe) {
Expand Down

0 comments on commit f7920a8

Please sign in to comment.