Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Convert to new _FIKE_OFFSET_BITS=64 changes.

Various fixes for failure to classify images.
Various fixes to loging.
Various other fixes.
Remove bad locks for referrer table.
  • Loading branch information...
commit a58c8348b4734664c46390f3dc78604093b4a6a4 1 parent 5c75592
@treveradams authored
View
2  configure.in
@@ -159,7 +159,7 @@ AC_CHECK_LIB(cxcore, cvCreateImage, [OPENCV_ADD_LDFLAG="-lcv -lcxcore -lhighgui"
AC_MSG_CHECKING([for OpenCV 2.2.x or newer])
AC_CHECK_LIB(opencv_core, cvCreateImage, [OPENCV_ADD_LDFLAG="-lopencv_core -lopencv_highgui -lopencv_objdetect -lopencv_features2d"; opencv="yes"; AC_DEFINE(HAVE_OPENCV_22X, 1, [Define HAVE_OPENCV if OpenCV installed])])
AC_MSG_CHECKING([for OpenCV 2.3.x or newer])
-AC_CHECK_LIB(opencv_imgproc, cvResize, [OPENCV_ADD_LDFLAG="-lopencv_core -lopencv_highgui -lopencv_objdetect -lopencv_features2d -lopencv_imgproc"; opencv="yes"; AC_DEFINE(HAVE_OPENCV_22X, 1, [Define HAVE_OPENCV if 2.3.x or newer OpenCV installed])])
+AC_CHECK_LIB(opencv_imgproc, cvResize, [OPENCV_ADD_LDFLAG="-lopencv_core -lopencv_highgui -lopencv_objdetect -lopencv_features2d -lopencv_imgproc"; opencv="yes"; AC_DEFINE(HAVE_OPENCV_23X, 1, [Define HAVE_OPENCV if 2.3.x or newer OpenCV installed])])
AC_SUBST(OPENCV_ADD_LDFLAG)
CFLAGS=$OLD_CFLAGS
LIBS=$OLD_LIBS
View
8 services/classify/bayes.c
@@ -22,8 +22,12 @@
#define _GNU_SOURCE
#endif
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
#endif
#include <stdio.h>
View
9 services/classify/fhs_findtolearn.c
@@ -19,7 +19,14 @@
#define _GNU_SOURCE
-#define _LARGEFILE64_SOURCE
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
#define NOT_CICAP
View
9 services/classify/fhs_judge.c
@@ -19,7 +19,14 @@
#define _GNU_SOURCE
-#define _LARGEFILE64_SOURCE
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
#define NOT_CICAP
View
9 services/classify/fhs_learn.c
@@ -19,7 +19,14 @@
#define _GNU_SOURCE
-#define _LARGEFILE64_SOURCE
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
#define NOT_CICAP
View
9 services/classify/fhs_makepreload.c
@@ -19,7 +19,14 @@
#define _GNU_SOURCE
-#define _LARGEFILE64_SOURCE
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
#define NOT_CICAP
View
9 services/classify/fnb_findtolearn.c
@@ -19,7 +19,14 @@
#define _GNU_SOURCE
-#define _LARGEFILE64_SOURCE
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
#define NOT_CICAP
View
9 services/classify/fnb_judge.c
@@ -19,7 +19,14 @@
#define _GNU_SOURCE
-#define _LARGEFILE64_SOURCE
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
#define NOT_CICAP
View
9 services/classify/fnb_learn.c
@@ -19,7 +19,14 @@
#define _GNU_SOURCE
-#define _LARGEFILE64_SOURCE
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
#ifndef NOT_AUTOCONF
#include "autoconf.h"
View
9 services/classify/fnb_makepreload.c
@@ -19,7 +19,14 @@
#define _GNU_SOURCE
-#define _LARGEFILE64_SOURCE
+
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
#define NOT_CICAP
View
8 services/classify/html.c
@@ -22,8 +22,12 @@
#define _GNU_SOURCE
#endif
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
#endif
#include <stdio.h>
View
8 services/classify/hyperspace.c
@@ -22,8 +22,12 @@
#define _GNU_SOURCE
#endif
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
+#endif
+#if (_FILE_OFFSET_BITS != 64)
+#undef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 64
#endif
#include <stdio.h>
View
31 services/classify/srv_classify.c
@@ -283,7 +283,6 @@ int srvclassify_init_service(ci_service_xdata_t * srv_xdata,
ci_thread_rwlock_wrlock(&imageclassify_rwlock);
ci_thread_rwlock_init(&referrers_rwlock);
- ci_thread_rwlock_wrlock(&referrers_rwlock);
createReferrerTable();
#endif
@@ -293,9 +292,9 @@ int srvclassify_init_service(ci_service_xdata_t * srv_xdata,
externalclassifytypes = calloc(ci_magic_types_num(magic_db), sizeof(external_conversion_t));
for (i = 0; i < ci_magic_types_num(magic_db); i++)
- classifytypes[i] = 0;
+ classifytypes[i] = NO_CLASSIFY;
for (i = 0; i < ci_magic_groups_num(magic_db); i++)
- classifygroups[i] = 0;
+ classifygroups[i] = NO_CLASSIFY;
ci_debug_printf(10, "Going to initialize srv_classify\n");
srv_classify_xdata = srv_xdata; /* Needed by db_reload command */
@@ -314,7 +313,7 @@ int srvclassify_init_service(ci_service_xdata_t * srv_xdata,
//#endif
initHTML();
ci_thread_rwlock_unlock(&textclassify_rwlock);
-#if defined HAVE_OPENCV
+#if defined(HAVE_OPENCV) || defined(HAVE_OPENCV_22X)
ci_thread_rwlock_unlock(&imageclassify_rwlock);
#endif
@@ -454,8 +453,8 @@ int srvclassify_check_preview_handler(char *preview_data, int preview_data_len,
#if defined(HAVE_OPENCV) || defined(HAVE_OPENCV_22X)
data->type_name = ci_data_type_name(magic_db, data->file_type);
#endif
- if ((data->must_classify = must_classify(data->file_type, data)) == 0) {
- ci_debug_printf(8, "Not in \"must classify list\". Allow it...... \n");
+ if ((data->must_classify = must_classify(data->file_type, data)) == NO_CLASSIFY) {
+ ci_debug_printf(8, "srv_classify: Not in \"must classify list\". Allow it...... \n");
return CI_MOD_ALLOW204;
}
@@ -467,7 +466,7 @@ int srvclassify_check_preview_handler(char *preview_data, int preview_data_len,
strstr(content_type, "text/javascript") || strstr(content_type, "text/jscript") ||
strstr(content_type, "text/css"))
{
- ci_debug_printf(8, "Non-content MIME type (%s). Allow it......\n", content_type);
+ ci_debug_printf(8, "srv_classify: Non-content MIME type (%s). Allow it......\n", content_type);
data->must_classify = NO_CLASSIFY; // these are not likely to contain data and confuse our classifier
return CI_MOD_ALLOW204;
}
@@ -483,8 +482,8 @@ int srvclassify_check_preview_handler(char *preview_data, int preview_data_len,
if (data->args.sizelimit && MAX_OBJECT_SIZE
&& content_size > MAX_OBJECT_SIZE) {
ci_debug_printf(1,
- "Object size is %" PRINTF_OFF_T "."
- " Bigger than max scannable file size (%"
+ "srv_classify: Object size is %" PRINTF_OFF_T "."
+ " Bigger than max classifiable file size (%"
PRINTF_OFF_T "). Allow it.... \n", content_size,
MAX_OBJECT_SIZE);
return CI_MOD_ALLOW204;
@@ -514,7 +513,7 @@ int srvclassify_read_from_net(char *buf, int len, int iseof, ci_request_t * req)
/* FIXME the following if should just process what is had at the moment... possible? */
if (ci_simple_file_size(data->body) >= MAX_OBJECT_SIZE && MAX_OBJECT_SIZE) {
- ci_debug_printf(1, "Object size is bigger than max scannable file size\n");
+ ci_debug_printf(1, "srv_classify: Object size is bigger than max classifiable file size\n");
data->must_classify = 0;
ci_req_unlock_data(req); /*Allow ICAP to send data before receives the EOF....... */
ci_simple_file_unlock_all(data->body); /*Unlock all body data to continue send them..... */
@@ -612,7 +611,7 @@ int srvclassify_end_of_data_handler(ci_request_t * req)
}
#endif
else if (data->allow204 && !ci_req_sent_data(req)) {
- ci_debug_printf(7, "srvClassify module: Respond with allow 204\n");
+ ci_debug_printf(7, "srv_classify module: Respond with allow 204\n");
return CI_MOD_ALLOW204;
}
@@ -1247,7 +1246,7 @@ int must_classify(int file_type, classify_req_data_t * data)
if (type == NO_CLASSIFY)
type = externalclassifytypes[file_type].data_type;
- else if (type == NO_CLASSIFY)
+ if (type == NO_CLASSIFY)
type = classifytypes[file_type];
ci_thread_rwlock_unlock(&textclassify_rwlock);
@@ -1551,9 +1550,9 @@ int cfg_ExternalTextConversion(char *directive, char **argv, void *setdata)
if (strcmp(directive, "ExternalTextFileType") == 0)
{
if(strcmp(argv[0], "STDOUT") == 0)
- type = EXTERNAL_TEXT_PIPE;
+ type |= EXTERNAL_TEXT_PIPE;
else if(strcmp(argv[0], "FILE") == 0)
- type = EXTERNAL_TEXT;
+ type |= EXTERNAL_TEXT;
else {
ci_debug_printf(1, "Incorrect second argument in directive:%s\n", directive);
ci_debug_printf(1, "Format: %s (STDOUT|FILE) FILE_TYPE PROGRAM ARG1 ARG2 ARG3 ...\n", directive);
@@ -1569,7 +1568,7 @@ int cfg_ExternalTextConversion(char *directive, char **argv, void *setdata)
{
if(externalclassifytypes[id].data_type & type)
{
- ci_debug_printf(1, "%s: already configurative to handle %s\n", directive, argv[1]);
+ ci_debug_printf(1, "%s: already configured to handle %s\n", directive, argv[1]);
return 0;
}
else externalclassifytypes[id].data_type |= type;
@@ -1620,7 +1619,7 @@ int cfg_ExternalImageConversion(char *directive, char **argv, void *setdata)
{
if(externalclassifytypes[id].data_type & type)
{
- ci_debug_printf(1, "%s: already configurative to handle %s\n", directive, argv[0]);
+ ci_debug_printf(1, "%s: already configured to handle %s\n", directive, argv[0]);
return 0;
}
else externalclassifytypes[id].data_type |= type;
View
46 services/classify/srv_classify_image.c
@@ -115,7 +115,7 @@ char old_dir[PATH_MAX];
if ((dirp = opendir(directory)) == NULL)
{
- ci_debug_printf(3, "unlink_directory: couldn't open '%s'", directory);
+ ci_debug_printf(3, "srv_classify_image: unlink_directory: couldn't open '%s'", directory);
return -1;
}
@@ -208,7 +208,7 @@ int i, j;
if((r1Area * current->category->coalesceOverlap <= over_width*over_height) || (r2Area * current->category->coalesceOverlap <= over_width*over_height))
{
- ci_debug_printf(10, "Merging detected %s at X: %d, Y: %d, Height: %d, Width: %d and X2: %d, Y2: %d, Height2: %d, Width2: %d\n",
+ ci_debug_printf(10, "srv_classify_image: Merging detected %s at X: %d, Y: %d, Height: %d, Width: %d and X2: %d, Y2: %d, Height2: %d, Width2: %d\n",
current->category->name, r->x, r->y, r->height, r->width, r2->x, r2->y, r2->height, r2->width);
tempRect = cvMaxRect( (CvRect*)cvGetSeqElem( newDetected, merged[i] ), r2);
cvSeqRemove( newDetected, merged[i] );
@@ -349,7 +349,7 @@ int i;
snprintf(fname, CI_MAX_PATH, "%s/detected-%s-%s-%dx%dx%dx%d.png", CLASSIFY_TMP_DIR, current->category->name, (rindex(mySession->fname, '/'))+1,
r->x, r->y, r->height, r->width);
fname[CI_MAX_PATH] = '\0';
- ci_debug_printf(8, "Saving detected object to: %s\n", fname);
+ ci_debug_printf(8, "srv_classify_image: Saving detected object to: %s\n", fname);
cvSetImageROI(mySession->origImage, *r);
cvSaveImage(fname, mySession->origImage, png_p);
cvResetImageROI(mySession->origImage);
@@ -392,7 +392,7 @@ int i;
radius = cvRound((r->width + r->height)*0.25);
cvCircle( mySession->origImage, center, radius, current->category->Color, 3, 8, 0 );
- ci_debug_printf(10, "%s Detected at X: %d, Y: %d, Height: %d, Width: %d\n", current->category->name, r->x, r->y, r->height, r->width);
+ ci_debug_printf(10, "srv_classify_image: %s Detected at X: %d, Y: %d, Height: %d, Width: %d\n", current->category->name, r->x, r->y, r->height, r->width);
}
current = current->next;
}
@@ -403,7 +403,7 @@ static void saveOriginal(ImageSession *mySession)
char fname[CI_MAX_PATH + 1];
snprintf(fname, CI_MAX_PATH, "%s/original-%s.png", CLASSIFY_TMP_DIR, (rindex(mySession->fname, '/'))+1);
fname[CI_MAX_PATH] = '\0';
- ci_debug_printf(10, "Saving Original Image To: %s\n", fname);
+ ci_debug_printf(10, "srv_classify_image: Saving Original Image To: %s\n", fname);
cvSaveImage(fname, mySession->origImage, png_p);
}
@@ -412,7 +412,7 @@ static void saveMarked(ImageSession *mySession)
char fname[CI_MAX_PATH + 1];
snprintf(fname, CI_MAX_PATH, "%s/marked-%s.png", CLASSIFY_TMP_DIR, (rindex(mySession->fname, '/'))+1);
fname[CI_MAX_PATH] = '\0';
- ci_debug_printf(10, "Saving Marked Image To: %s\n", fname);
+ ci_debug_printf(10, "srv_classify_image: Saving Marked Image To: %s\n", fname);
cvSaveImage(fname, mySession->origImage, png_p);
}
@@ -422,16 +422,20 @@ struct stat stat_buf;
classify_req_data_t *data = ci_service_data(mySession->req);
char imageFILENAME[CI_MAX_PATH + 1];
- ci_http_response_remove_header(mySession->req, "Content-Length");
+ ci_http_response_remove_header(mySession->req, "Content-Length");
data->body->readpos = 0;
close(data->body->fd);
unlink(data->body->filename);
snprintf(imageFILENAME, CI_MAX_PATH, "%s.%s", data->body->filename, data->type_name);
imageFILENAME[CI_MAX_PATH] = '\0';
- if(strstr(imageFILENAME, "jpg"))
+ if(strstr(imageFILENAME, "jpg"))
cvSaveImage(imageFILENAME, mySession->origImage, jpeg_p);
- else
+ else
+ {
cvSaveImage(imageFILENAME, mySession->origImage, png_p);
+ ci_http_response_remove_header(mySession->req, "Content-Type");
+ ci_http_response_add_header(mySession->req, "Content-Type: image/png");
+ }
snprintf(data->body->filename, CI_FILENAME_LEN + 1, "%s", imageFILENAME);
data->body->filename[CI_FILENAME_LEN] = '\0';
data->body->fd = open(data->body->filename, O_RDWR | O_EXCL, F_PERM);
@@ -440,7 +444,7 @@ char imageFILENAME[CI_MAX_PATH + 1];
// Make new content length header
snprintf(imageFILENAME, CI_MAX_PATH, "Content-Length: %ld", (long) data->body->endpos);
imageFILENAME[CI_MAX_PATH] = '\0';
- ci_http_response_add_header(mySession->req, imageFILENAME);
+ ci_http_response_add_header(mySession->req, imageFILENAME);
}
static void getRightSize(ImageSession *mySession)
@@ -466,10 +470,10 @@ int maxDim = mySession->origImage->width > mySession->origImage->height ? mySess
else cvCvtColor( mySession->origImage, mySession->rightImage, CV_BGR2GRAY );
//cvEqualizeHist( small_img, small_img ); // Removed as it does weird things and drops accuracy
- if(mySession->scale != 1.0f) ci_debug_printf(9, "Image Resized - X: %d, Y: %d (shrunk by a factor of %f)\n", mySession->rightImage->width, mySession->rightImage->height, mySession->scale);
+ if(mySession->scale != 1.0f) ci_debug_printf(9, "srv_classify_image: Image Resized - X: %d, Y: %d (shrunk by a factor of %f)\n", mySession->rightImage->width, mySession->rightImage->height, mySession->scale);
t = cvGetTickCount() - t;
t = t / ((double)cvGetTickFrequency() * 1000.);
- ci_debug_printf(8, "Scaling and Color Prep took: %gms.\n", t);
+ ci_debug_printf(8, "srv_classify_image: Scaling and Color Prep took: %gms.\n", t);
}
LinkedCascade *getFreeCascade(ImageCategory *category)
@@ -491,7 +495,7 @@ int attempts = 0;
// Add item to busy list
item->next = category->busy_cascade;
category->busy_cascade = item;
- if(attempts >= 4) ci_debug_printf(3, "Had to wait on cascade %s, consider increasing ImageCategoryCopies in configuration. Currently set to %d. Retried %d times.\n", category->name, IMAGE_CATEGORY_COPIES, attempts);
+ if(attempts >= 4) ci_debug_printf(3, "srv_classify_image: Had to wait on cascade %s, consider increasing ImageCategoryCopies in configuration. Currently set to %d. Retried %d times.\n", category->name, IMAGE_CATEGORY_COPIES, attempts);
}
}
else {
@@ -558,7 +562,7 @@ LinkedCascade *cascade;
}
t = cvGetTickCount() - t;
t = t / ((double)cvGetTickFrequency() * 1000.);
- ci_debug_printf(8, "File: %s Object: %s (%d) Detection took: %gms.\n", (rindex(mySession->fname, '/'))+1, current->category->name, current->detected->total, t);
+ ci_debug_printf(8, "srv_classify_image: File: %s Object: %s (%d) Detection took: %gms.\n", (rindex(mySession->fname, '/'))+1, current->category->name, current->detected->total, t);
current = current->next;
}
@@ -610,7 +614,7 @@ int copies;
new_cascade->cascade = (CvHaarClassifierCascade*) cvLoad(current->cascade_location, NULL, NULL, NULL );
if( !new_cascade->cascade )
{
- ci_debug_printf(3, "Failed to load cascade for %s\n", current->name);
+ ci_debug_printf(3, "srv_classify_image: Failed to load cascade for %s\n", current->name);
free(new_cascade);
copies = IMAGE_CATEGORY_COPIES;
if(copies == 0)
@@ -625,7 +629,7 @@ int copies;
}
else
{
- ci_debug_printf(10, "Successfully loaded cascade for %s\n", current->name);
+ ci_debug_printf(10, "srv_classify_image: Successfully loaded cascade for %s\n", current->name);
new_cascade->next = current->free_cascade;
current->free_cascade = new_cascade;
}
@@ -683,7 +687,7 @@ ImageDetected *nDetected = NULL, *cDetected = NULL;
// If there are no categories, we cannot detect anything
if(current == NULL)
{
- ci_debug_printf(3, "No categories present. I cannot initiate session.\n");
+ ci_debug_printf(3, "srv_classify_image: No categories present. I cannot initiate session.\n");
return NO_CATEGORIES;
}
@@ -791,7 +795,7 @@ int minDim = 0, ret = 0;
minDim = mySession.origImage->width < mySession.origImage->height ? mySession.origImage->width : mySession.origImage->height;
if(ImageMinProcess >= minDim || minDim < 5)
{
- ci_debug_printf(10, "Image too small for classification per configuration and/or sanity, letting pass.\n");
+ ci_debug_printf(10, "srv_classify_image: Image too small for classification per configuration and/or sanity, letting pass.\n");
cvReleaseImage(&mySession.origImage);
return 0; // Image is too small, let it pass
}
@@ -843,7 +847,7 @@ int minDim = 0, ret = 0;
ci_thread_rwlock_unlock(&imageclassify_rwlock);
}
else {
- ci_debug_printf(8, "Could not load image (%s) for classification.\n", data->body->filename);
+ ci_debug_printf(8, "srv_classify_image: Could not load image (%s) for classification.\n", data->body->filename);
return -1;
}
@@ -899,7 +903,7 @@ ImageCategory *current_category = imageCategories;
else if(child_pid < 0)
{
// Error condition
- ci_debug_printf(3, "categorize_external_image: failed to fork\n");
+ ci_debug_printf(3, "categorize_external_image: categorize_external_image: failed to fork\n");
}
else { // We are the original
waitpid(child_pid, &wait_status, 0);
@@ -909,7 +913,7 @@ ImageCategory *current_category = imageCategories;
if ((dirp = opendir(data->external_body->filename)) == NULL)
{
- ci_debug_printf(3, "categorize_external_image: couldn't open '%s'", data->external_body->filename);
+ ci_debug_printf(3, "srv_classify_image: categorize_external_image: couldn't open '%s'", data->external_body->filename);
return -1;
}
Please sign in to comment.
Something went wrong with that request. Please try again.