diff --git a/CMakeLists.txt b/CMakeLists.txt index cf377ddd..e31993a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,11 +22,12 @@ # 5) build the project "PACKAGE" to create an NSIS-installer (NSIS is required) -project(OpenTLD) +project(OpenTLD-OCL) cmake_minimum_required(VERSION 2.6) find_package(OpenCV REQUIRED) +find_package(OpenCL REQUIRED) if(BUILD_QOPENTLD) find_package(Qt4 REQUIRED) diff --git a/OpenTLDConfig.cmake.in b/OpenTLDConfig.cmake.in index 0d941c2e..5fcec247 100644 --- a/OpenTLDConfig.cmake.in +++ b/OpenTLDConfig.cmake.in @@ -1,5 +1,6 @@ -find_package(OpenCV) +find_package(OpenCV) +find_package(OpenCL) set(bin_dir "@PROJECT_BINARY_DIR@") set(src_dir "@PROJECT_SOURCE_DIR@") diff --git a/README.md b/README.md index a46ea226..9ff0ce46 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ This is a C++ implementation of OpenTLD that was originally published in MATLAB by Zdenek Kalal. OpenTLD is used for tracking objects in video streams. What makes this algorithm outstanding is that it does not make use of any training data. This implementation is based solely on open source libraries, meaning that you do not need any commercial products to compile or run it. -The easiest way to get started is to download the precompiled [binaries](https://github.com/gnebehay/OpenTLD/releases) that are available for Windows and Ubuntu 12.04. +GPU is a good accelerated compute resource for speed up Tracking speed, so I want to complete one OpenCL version for OpenTLD, I will begin today. + +The easiest way to get started is to download the precompiled [binaries](https://github.com/gnebehay/OpenTLD/releases) that are available for Windows and Ubuntu 12.04. 聽 There is also a [PPA](https://launchpad.net/~opentld/+archive/ppa) available for Ubuntu 12.04. You just have to execute these commands: ``` sudo add-apt-repository ppa:opentld/ppa diff --git a/src/libopentld/CMakeLists.txt b/src/libopentld/CMakeLists.txt index 740c07d2..193c9297 100644 --- a/src/libopentld/CMakeLists.txt +++ b/src/libopentld/CMakeLists.txt @@ -3,8 +3,10 @@ include_directories(imacq mftracker tld ../3rdparty/cvblobs - ${OpenCV_INCLUDE_DIRS}) -link_directories(${OpenCV_LIB_DIR}) + ${OpenCV_INCLUDE_DIRS} + ${OpenCL_INCLUDE_DIRS}) +link_directories(${OpenCV_LIB_DIR} +${OpenCL_LIB_DIR}) add_library(libopentld imacq/ImAcq.cpp @@ -40,9 +42,10 @@ add_library(libopentld tld/NormalizedPatch.h tld/TLD.h tld/TLDUtil.h - tld/VarianceFilter.h) + tld/VarianceFilter.h + tld/switch.h) -target_link_libraries(libopentld ${OpenCV_LIBS}) +target_link_libraries(libopentld ${OpenCV_LIBS} ${OPENCL_LIBRARIES}) set_target_properties(libopentld PROPERTIES OUTPUT_NAME opentld) diff --git a/src/libopentld/imacq/ImAcq.cpp b/src/libopentld/imacq/ImAcq.cpp index 24d44410..9cd33467 100644 --- a/src/libopentld/imacq/ImAcq.cpp +++ b/src/libopentld/imacq/ImAcq.cpp @@ -42,7 +42,8 @@ static void msleep(int milliseconds) ImAcq *imAcqAlloc() { ImAcq *imAcq = (ImAcq *)malloc(sizeof(ImAcq)); - imAcq->method = IMACQ_CAM; + // imAcq->method = IMACQ_CAM; + imAcq->method = 2; imAcq->currentFrame = 1; imAcq->lastFrame = 0; imAcq->camNo = 0; @@ -52,6 +53,7 @@ ImAcq *imAcqAlloc() void imAcqInit(ImAcq *imAcq) { + // imAcq->method = 2; if(imAcq->method == IMACQ_CAM) { imAcq->capture = cvCaptureFromCAM(imAcq->camNo); @@ -64,7 +66,8 @@ void imAcqInit(ImAcq *imAcq) } else if(imAcq->method == IMACQ_VID) { - imAcq->capture = cvCaptureFromAVI(imAcq->imgPath); + imAcq->capture = cvCaptureFromAVI(imAcq->imgPath); + //imAcq->capture = cvCaptureFromAVI("D:\\OpenTLD-master\\build\\bin\\Release\\IMG_5903.MOV"); if(imAcq->capture == NULL) { @@ -283,3 +286,4 @@ int imAcqVidGetNumberOfFrames(ImAcq *imAcq) { return ((int) cvGetCaptureProperty(imAcq->capture , CV_CAP_PROP_FRAME_COUNT)); } + diff --git a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp new file mode 100644 index 00000000..6a8b6a15 --- /dev/null +++ b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp @@ -0,0 +1,449 @@ + + +#pragma OPENCL EXTENSION cl_amd_printf : enable + +#define TLD_PATCH_SIZE 15 + + +__kernel void nnClassifier( + __global float * NNResultsArray, + const unsigned int truePostiveSize, + const unsigned int falsePositiveSize, + const unsigned int CandidatesToNNClassifySize, + __global float * truePostiveData, + __global float * FalsePostiveData, + __global float * CandidatesToNNClassifyPatches) +{ + + int ThreadID = get_global_id(0); //ThreadID is one of truePostiveSize or falsePositiveSize + + __global float *pSrc1, *pSrc2; + int patchSize = TLD_PATCH_SIZE * TLD_PATCH_SIZE; + int CandidatesSize = CandidatesToNNClassifySize; + __global float *conf; + + pSrc2 = CandidatesToNNClassifyPatches; //TLD_PATCH_SIZE=15 + conf = NNResultsArray + ThreadID *CandidatesSize ; + if (ThreadID < truePostiveSize && ThreadID < (truePostiveSize + falsePositiveSize)) + { + pSrc1 = truePostiveData + ThreadID * patchSize; //TLD_PATCH_SIZE=15 + for (int j = 0; j < CandidatesSize; j++) + { + double corr = 0; + double norm1 = 0; + double norm2 = 0; + for (int i = 0; i < patchSize; i++) + { + corr += pSrc1[i] * pSrc2[CandidatesSize*j + i]; + norm1 += pSrc1[i] * pSrc1[i]; + norm2 += pSrc2[j*CandidatesSize + i] * pSrc2[CandidatesSize*j + i]; + } + // normalization to <0,1> //return (corr / sqrt(norm1 * norm2) + 1) / 2.0; + + if (norm1 == 0.0 || norm2 == 0.0) + { + norm1 = 1.0; + norm2 = 1.0; + } + else + *conf++ = (corr / sqrt(norm1 * norm2) + 1) / 2.0; + } + } + else if(ThreadID > truePostiveSize && ThreadID < (truePostiveSize + falsePositiveSize)) + { + pSrc1 = FalsePostiveData+ (ThreadID- truePostiveSize) * patchSize; //TLD_PATCH_SIZE=15 + for (int j = 0; j < CandidatesSize; j++) + { + double corr = 0; + double norm1 = 0; + double norm2 = 0; + for (int i = 0; i < patchSize; i++) + { + corr += pSrc1[i] * pSrc2[i]; + norm1 += pSrc1[i] * pSrc1[i]; + norm2 += pSrc2[i] * pSrc2[i]; + } + // normalization to <0,1> //return (corr / sqrt(norm1 * norm2) + 1) / 2.0; + if(norm1==0.0 || norm2 ==0.0) + { + norm1 = 1.0; + norm2 = 1.0; + } + else + *conf++ = (corr / sqrt(norm1 * norm2) + 1) / 2.0; + + } + } + // tld_window_size = 5 + + //resize(image(source(x, y, width, height), 15 * 15, patchValue); //opencv resize function with bilinear method. + +} + + +__kernel void EnsembleClassifierFilter( + const int numIndices, + const unsigned int numTrees, + const unsigned int numFeatures, + const unsigned int TLD_WINDOW_OFFSET_SIZE, + __global int* oclbuffWindowsOffset, + __global uchar * img, + __global int* featureOffsets, + __global float* oclbuffDetectionResultfeatureVectors, + __global float* oclbuffDetectionResultPosteriors, + __global float* posteriors, + __global int* oclbuffII, + __global float* oclbuffIISqure, + __global float* oclbuffDetectionResultVarious, + const float minVar, + const int numWindows + + ) +{ + __global int *off, *bbox; + __global int * featureVector; + int bbx1,bbx2,bbx3,bbx4; + + + int index,bbox4 ; + float conf=0.0; + int fp0,fp1; + int windowIdx = get_global_id(0); + int gsize = get_global_size(0); + if(windowIdx fp1) + { + index |= 1; + } + off += 2; + } + + featureVector[treeIdx] = index; + + conf = conf+ posteriors[treeIdx * numIndices + index]; + + } + + oclbuffDetectionResultPosteriors[windowIdx] = conf; +// if(conf<0.5 ) +// windowFlags[windowIdx] = 2; +// else +// windowFlags[windowIdx] = 0; + + // if(windowIdx==753666) + // printf("conf=%f,windowFlags[windowIdx]=%d...... \n",conf,windowFlags[windowIdx]); + // if(conf>0.5f) + +Endofkernel: + ; + } +} + +#ifdef DOUBLE_SUPPORT +#ifdef cl_amd_fp64 +#pragma OPENCL EXTENSION cl_amd_fp64:enable +#elif defined (cl_khr_fp64) +#pragma OPENCL EXTENSION cl_khr_fp64:enable +#endif +#endif + +#define LSIZE 256 +#define LSIZE_1 255 +#define LSIZE_2 254 +#define HF_LSIZE 128 +#define LOG_LSIZE 8 +#define LOG_NUM_BANKS 5 +#define NUM_BANKS 32 +#define GET_CONFLICT_OFFSET(lid) ((lid) >> LOG_NUM_BANKS) + + +kernel void integral_cols_DD(__global uchar4 *src,__global int *sum ,__global float *sqsum, + int src_offset,int pre_invalid,int rows,int cols,int src_step,int dst_step) +{ + +// printf("src_offset = %d\n pre_invalid=%d\n,rows=%d\n,cols=%d\n,src_step=%d,dst_step=%d\n",src_offset,pre_invalid,rows,cols,src_step,dst_step); + + int lid = get_local_id(0); + int gid = get_group_id(0); + int4 src_t[2], sum_t[2]; + float4 sqsum_t[2]; + __local int4 lm_sum[2][LSIZE + LOG_LSIZE]; + __local float4 lm_sqsum[2][LSIZE + LOG_LSIZE]; + __local int* sum_p; + __local float* sqsum_p; + src_step = src_step >> 2; + gid = gid << 1; + for(int i = 0; i < rows; i =i + LSIZE_1) + { + src_t[0] = (i + lid < rows ? convert_int4(src[src_offset + (lid+i) * src_step + min(gid, cols - 1)]) : 0); + src_t[1] = (i + lid < rows ? convert_int4(src[src_offset + (lid+i) * src_step + min(gid + 1, cols - 1)]) : 0); + + sum_t[0] = (i == 0 ? 0 : lm_sum[0][LSIZE_2 + LOG_LSIZE]); + sqsum_t[0] = (i == 0 ? (float4)0 : lm_sqsum[0][LSIZE_2 + LOG_LSIZE]); + sum_t[1] = (i == 0 ? 0 : lm_sum[1][LSIZE_2 + LOG_LSIZE]); + sqsum_t[1] = (i == 0 ? (float4)0 : lm_sqsum[1][LSIZE_2 + LOG_LSIZE]); + barrier(CLK_LOCAL_MEM_FENCE); + + int bf_loc = lid + GET_CONFLICT_OFFSET(lid); + lm_sum[0][bf_loc] = src_t[0]; + lm_sqsum[0][bf_loc] = convert_float4(src_t[0] * src_t[0]); + + lm_sum[1][bf_loc] = src_t[1]; + lm_sqsum[1][bf_loc] = convert_float4(src_t[1] * src_t[1]); + + int offset = 1; + for(int d = LSIZE >> 1 ; d > 0; d>>=1) + { + barrier(CLK_LOCAL_MEM_FENCE); + int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; + ai += GET_CONFLICT_OFFSET(ai); + bi += GET_CONFLICT_OFFSET(bi); + + if((lid & 127) < d) + { + lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; + lm_sqsum[lid >> 7][bi] += lm_sqsum[lid >> 7][ai]; + } + offset <<= 1; + } + barrier(CLK_LOCAL_MEM_FENCE); + if(lid < 2) + { + lm_sum[lid][LSIZE_2 + LOG_LSIZE] = 0; + lm_sqsum[lid][LSIZE_2 + LOG_LSIZE] = 0; + } + for(int d = 1; d < LSIZE; d <<= 1) + { + barrier(CLK_LOCAL_MEM_FENCE); + offset >>= 1; + int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; + ai += GET_CONFLICT_OFFSET(ai); + bi += GET_CONFLICT_OFFSET(bi); + + if((lid & 127) < d) + { + lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; + lm_sum[lid >> 7][ai] = lm_sum[lid >> 7][bi] - lm_sum[lid >> 7][ai]; + + lm_sqsum[lid >> 7][bi] += lm_sqsum[lid >> 7][ai]; + lm_sqsum[lid >> 7][ai] = lm_sqsum[lid >> 7][bi] - lm_sqsum[lid >> 7][ai]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + int loc_s0 = gid * dst_step + i + lid - 1 - pre_invalid * dst_step / 4, loc_s1 = loc_s0 + dst_step ; + if(lid > 0 && (i+lid) <= rows) + { + lm_sum[0][bf_loc] += sum_t[0]; + lm_sum[1][bf_loc] += sum_t[1]; + lm_sqsum[0][bf_loc] += sqsum_t[0]; + lm_sqsum[1][bf_loc] += sqsum_t[1]; + sum_p = (__local int*)(&(lm_sum[0][bf_loc])); + sqsum_p = (__local float*)(&(lm_sqsum[0][bf_loc])); + for(int k = 0; k < 4; k++) + { + if(gid * 4 + k >= cols + pre_invalid || gid * 4 + k < pre_invalid) continue; + sum[loc_s0 + k * dst_step / 4] = sum_p[k]; + sqsum[loc_s0 + k * dst_step / 4] = sqsum_p[k]; + + } + sum_p = (__local int*)(&(lm_sum[1][bf_loc])); + sqsum_p = (__local float*)(&(lm_sqsum[1][bf_loc])); + for(int k = 0; k < 4; k++) + { + if(gid * 4 + k + 4 >= cols + pre_invalid) break; + sum[loc_s1 + k * dst_step / 4] = sum_p[k]; + sqsum[loc_s1 + k * dst_step / 4] = sqsum_p[k]; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + } + // if(lid==0 && gid==0) + // printf("sum_start=%d,sqsum_start=%f\n",sum[lid],sqsum[lid] ); + +} + + +kernel void integral_rows_DD(__global int4 *srcsum,__global float4 * srcsqsum,__global int *sum , + __global float *sqsum,int rows,int cols,int src_step,int sum_step, + int sqsum_step,int sum_offset,int sqsum_offset) +{ + + int lid = get_local_id(0); + int gid = get_group_id(0); + int4 src_t[2], sum_t[2]; + float4 sqsrc_t[2],sqsum_t[2]; + __local int4 lm_sum[2][LSIZE + LOG_LSIZE]; + __local float4 lm_sqsum[2][LSIZE + LOG_LSIZE]; + __local int *sum_p; + __local float *sqsum_p; + + + // if(lid==0 && gid==0) + // printf("sum_p=%d,sqsum_p=%f\n",srcsum[0].x,srcsqsum[0].x ); + + src_step = src_step >> 4; + for(int i = 0; i < rows; i =i + LSIZE_1) + { + src_t[0] = i + lid < rows ? srcsum[(lid+i) * src_step + gid * 2] : (int4)0; + sqsrc_t[0] = i + lid < rows ? srcsqsum[(lid+i) * src_step + gid * 2] : (float4)0; + src_t[1] = i + lid < rows ? srcsum[(lid+i) * src_step + gid * 2 + 1] : (int4)0; + sqsrc_t[1] = i + lid < rows ? srcsqsum[(lid+i) * src_step + gid * 2 + 1] : (float4)0; + + sum_t[0] = (i == 0 ? 0 : lm_sum[0][LSIZE_2 + LOG_LSIZE]); + sqsum_t[0] = (i == 0 ? (float4)0 : lm_sqsum[0][LSIZE_2 + LOG_LSIZE]); + sum_t[1] = (i == 0 ? 0 : lm_sum[1][LSIZE_2 + LOG_LSIZE]); + sqsum_t[1] = (i == 0 ? (float4)0 : lm_sqsum[1][LSIZE_2 + LOG_LSIZE]); + barrier(CLK_LOCAL_MEM_FENCE); + + int bf_loc = lid + GET_CONFLICT_OFFSET(lid); + lm_sum[0][bf_loc] = src_t[0]; + lm_sqsum[0][bf_loc] = sqsrc_t[0]; + + lm_sum[1][bf_loc] = src_t[1]; + lm_sqsum[1][bf_loc] = sqsrc_t[1]; + + int offset = 1; + for(int d = LSIZE >> 1 ; d > 0; d>>=1) + { + barrier(CLK_LOCAL_MEM_FENCE); + int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; + ai += GET_CONFLICT_OFFSET(ai); + bi += GET_CONFLICT_OFFSET(bi); + + if((lid & 127) < d) + { + lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; + lm_sqsum[lid >> 7][bi] += lm_sqsum[lid >> 7][ai]; + } + offset <<= 1; + } + barrier(CLK_LOCAL_MEM_FENCE); + if(lid < 2) + { + lm_sum[lid][LSIZE_2 + LOG_LSIZE] = 0; + lm_sqsum[lid][LSIZE_2 + LOG_LSIZE] = 0; + } + for(int d = 1; d < LSIZE; d <<= 1) + { + barrier(CLK_LOCAL_MEM_FENCE); + offset >>= 1; + int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; + ai += GET_CONFLICT_OFFSET(ai); + bi += GET_CONFLICT_OFFSET(bi); + + if((lid & 127) < d) + { + lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; + lm_sum[lid >> 7][ai] = lm_sum[lid >> 7][bi] - lm_sum[lid >> 7][ai]; + + lm_sqsum[lid >> 7][bi] += lm_sqsum[lid >> 7][ai]; + lm_sqsum[lid >> 7][ai] = lm_sqsum[lid >> 7][bi] - lm_sqsum[lid >> 7][ai]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); +// if(gid == 0 && (i + lid) <= rows) + // { + // sum[sum_offset + i + lid] = 0; + // sqsum[sqsum_offset + i + lid] = 0; + // } + // if(i + lid == 0) +// { + // int loc0 = gid * 2 * sum_step; + // int loc1 = gid * 2 * sqsum_step; + // for(int k = 1; k <= 8; k++) + // { + // if(gid * 8 + k > cols) break; + // sum[sum_offset + loc0 + k * sum_step / 4] = 0; + // sqsum[sqsum_offset + loc1 + k * sqsum_step / 4] = 0; + // } + // } + //copy from imgproc_integral_sum.cl + // int loc_s0 = sum_offset + gid * 2 * sum_step + i + lid -1 , loc_s1 = loc_s0 + sum_step ; + + + // int loc_s0 = sum_offset + gid * 2 * sum_step + sum_step / 4 + i + lid, loc_s1 = loc_s0 + sum_step ; + //int loc_sq0 = sqsum_offset + gid * 2 * sqsum_step + sqsum_step / 4 + i + lid, loc_sq1 = loc_sq0 + sqsum_step ; + int loc_s0 = sum_offset + gid * 2 * sum_step + i + lid - 1 , loc_s1 = loc_s0 + sum_step ; + int loc_sq0 = sqsum_offset + gid * 2 * sqsum_step + i + lid - 1, loc_sq1 = loc_sq0 + sqsum_step ; + if(lid > 0 && (i+lid) <= rows) + { + lm_sum[0][bf_loc] += sum_t[0]; + lm_sum[1][bf_loc] += sum_t[1]; + lm_sqsum[0][bf_loc] += sqsum_t[0]; + lm_sqsum[1][bf_loc] += sqsum_t[1]; + sum_p = (__local int*)(&(lm_sum[0][bf_loc])); + sqsum_p = (__local float*)(&(lm_sqsum[0][bf_loc])); + for(int k = 0; k < 4; k++) + { + if(gid * 8 + k >= cols) break; + sum[loc_s0 + k * sum_step / 4] = sum_p[k]; + sqsum[loc_sq0 + k * sqsum_step / 4] = sqsum_p[k]; + } + sum_p = (__local int*)(&(lm_sum[1][bf_loc])); + sqsum_p = (__local float*)(&(lm_sqsum[1][bf_loc])); + for(int k = 0; k < 4; k++) + { + if(gid * 8 + 4 + k >= cols) break; + sum[loc_s1 + k * sum_step / 4] = sum_p[k]; + sqsum[loc_sq1 + k * sqsum_step / 4] = sqsum_p[k]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + // if(lid==0 && gid==0) + // printf("sum=%d,sqsum=%f\n",sum[lid],sqsum[lid] ); +} + + + + diff --git a/src/libopentld/kernel/cltldOverlapRect_kernel.cpp b/src/libopentld/kernel/cltldOverlapRect_kernel.cpp new file mode 100644 index 00000000..55486fb9 --- /dev/null +++ b/src/libopentld/kernel/cltldOverlapRect_kernel.cpp @@ -0,0 +1,75 @@ + + + +#pragma OPENCL EXTENSION cl_amd_printf : enable + +__kernel void cltldOverlapRect( + const unsigned int bb10, + const unsigned int bb11, + const unsigned int bb12, + const unsigned int bb13, + __global int* oclbuffWindowsOffset, + __global float* overlap, + const unsigned int TLD_WINDOW_SIZE + ) +{ + int windowIdx = get_global_id(0); + int min,max,sum1,sum2; + __global int *bb2= oclbuffWindowsOffset + windowIdx * TLD_WINDOW_SIZE; + + if (bb10 > bb2[0] + bb2[2]) + { + overlap[windowIdx] = 0.0; + goto finished; + } + + if (bb11 > bb2[1] + bb2[3]) + { + overlap[windowIdx] = 0.0; + goto finished; + } + + if (bb10 + bb12 < bb2[0]) + { + overlap[windowIdx] = 0.0; + goto finished; + } + + if (bb11 + bb13 < bb2[1]) + { + overlap[windowIdx] = 0.0; + goto finished; + } + else + { + sum1 = bb10 + bb12; + sum2 = bb2[0] + bb2[2]; + min = sum1bb2[0]? bb10:bb2[0]; + int colInt =min-max; + + sum1 = bb11 + bb13; + sum2 = bb2[1] + bb2[3]; + min = sum1bb2[1]? bb11:bb2[1]; + int rowInt =min-max; + + int intersection = colInt * rowInt; + int area1 = bb12 * bb13; + int area2 = bb2[2] * bb2[3]; + overlap[windowIdx] = intersection / (float)(area1 + area2 - intersection); + } + + finished: + ; + //if(overlap[windowIdx]!=0.0) + //if( windowIdx==249404 ) + //{ + // for(int i=0;i<100;i++) + // printf(".....windowIdx =%d\n",windowIdx); + // printf("windowIdx=%d,overlap[xx]=%f\n",windowIdx,overlap[windowIdx]); + //} + } + + + \ No newline at end of file diff --git a/src/libopentld/kernel/varianceFilter_kernel.cpp b/src/libopentld/kernel/varianceFilter_kernel.cpp new file mode 100644 index 00000000..62230e2c --- /dev/null +++ b/src/libopentld/kernel/varianceFilter_kernel.cpp @@ -0,0 +1,46 @@ + + +#pragma OPENCL EXTENSION cl_amd_printf : enable + +__kernel void varianceFilter( + __global int* oclbuffWindowsOffset, + __global int* oclbuffII, + __global float* oclbuffIISqure, + __global float* oclbuffDetectionResultVarious, +__global float* oclbuffDetectionResultPosteriors, +const unsigned int numWindows, +const float minVar, +__global int* windowFlags) +{ + int windowIdx = get_global_id(0); + //__global int *off = &(oclbuffWindowsOffset[windowIdx*6]) ; + //__global int *off = oclbuffWindowsOffset; + float bboxvar=0.0; + float mX; + float mX2; + + + int off0 =oclbuffWindowsOffset[ windowIdx*6 + 0 ]; + int off1 =oclbuffWindowsOffset[ windowIdx*6 + 1 ]; + int off2 =oclbuffWindowsOffset[ windowIdx*6 + 2 ]; + int off3 =oclbuffWindowsOffset[ windowIdx*6 + 3 ]; + int off4 =oclbuffWindowsOffset[ windowIdx*6 + 4 ]; + int off5 =oclbuffWindowsOffset[ windowIdx*6 + 5 ]; + mX = ( oclbuffII[off3] - oclbuffII[off2] -oclbuffII[off1] + oclbuffII[off0] )/(float) off5; + mX2 = ( oclbuffIISqure[off3] - oclbuffIISqure[off2] -oclbuffIISqure[off1] + oclbuffIISqure[off0] )/(float) off5; + + bboxvar = mX2-mX*mX; + oclbuffDetectionResultVarious[windowIdx] = bboxvar; + + if(bboxvar < minVar) + { + oclbuffDetectionResultPosteriors[windowIdx] = 0.0; + windowFlags[windowIdx] = 1; + } + //windowFlags[windowIdx] = 100; + + +} + + + \ No newline at end of file diff --git a/src/libopentld/mftracker/BBPredict.cpp b/src/libopentld/mftracker/BBPredict.cpp index 9a2556e4..1cac725d 100644 --- a/src/libopentld/mftracker/BBPredict.cpp +++ b/src/libopentld/mftracker/BBPredict.cpp @@ -28,7 +28,6 @@ #include "BBPredict.h" #include -#include #include "Median.h" @@ -38,7 +37,7 @@ */ float getBbWidth(float *bb) { - return std::abs(bb[2] - bb[0] + 1); + return abs(bb[2] - bb[0] + 1); } /** * Returns hight of Boundingbox. @@ -46,7 +45,7 @@ float getBbWidth(float *bb) */ float getBbHeight(float *bb) { - return std::abs(bb[3] - bb[1] + 1); + return abs(bb[3] - bb[1] + 1); } /** * Calculates the new (moved and resized) Bounding box. diff --git a/src/libopentld/tld/DetectionResult.cpp b/src/libopentld/tld/DetectionResult.cpp index 1600d071..6f894a20 100644 --- a/src/libopentld/tld/DetectionResult.cpp +++ b/src/libopentld/tld/DetectionResult.cpp @@ -59,6 +59,9 @@ void DetectionResult::init(int numWindows, int numTrees) variances = new float[numWindows]; posteriors = new float[numWindows]; featureVectors = new int[numWindows * numTrees]; + windowFlags = new float[numWindows]; + for (int i = 0; i < numWindows; i++) + windowFlags[i] = 0.0f; delete confidentIndices; confidentIndices = new vector(); diff --git a/src/libopentld/tld/DetectionResult.h b/src/libopentld/tld/DetectionResult.h index 3ac15d1b..f3efb47f 100644 --- a/src/libopentld/tld/DetectionResult.h +++ b/src/libopentld/tld/DetectionResult.h @@ -43,6 +43,7 @@ class DetectionResult std::vector* confidentIndices; int *featureVectors; float *variances; + float *windowFlags; int numClusters; cv::Rect *detectorBB; //Contains a valid result only if numClusters = 1 diff --git a/src/libopentld/tld/DetectorCascade.cpp b/src/libopentld/tld/DetectorCascade.cpp index 16d7dbe1..9b6564c5 100644 --- a/src/libopentld/tld/DetectorCascade.cpp +++ b/src/libopentld/tld/DetectorCascade.cpp @@ -18,11 +18,11 @@ */ /* - * DetectorCascade.cpp - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* DetectorCascade.cpp +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #include "DetectorCascade.h" @@ -30,315 +30,645 @@ #include "TLDUtil.h" -using namespace cv; - -namespace tld -{ - -//TODO: Convert this to a function -#define sub2idx(x,y,imgWidthStep) ((int) (floor((x)+0.5) + floor((y)+0.5)*(imgWidthStep))) - -DetectorCascade::DetectorCascade() -{ - objWidth = -1; //MUST be set before calling init - objHeight = -1; //MUST be set before calling init - useShift = 1; - imgHeight = -1; - imgWidth = -1; - - shift = 0.1; - minScale = -10; - maxScale = 10; - minSize = 25; - imgWidthStep = -1; - - numTrees = 10; - numFeatures = 13; - - initialised = false; - - foregroundDetector = new ForegroundDetector(); - varianceFilter = new VarianceFilter(); - ensembleClassifier = new EnsembleClassifier(); - nnClassifier = new NNClassifier(); - clustering = new Clustering(); - - detectionResult = new DetectionResult(); -} - -DetectorCascade::~DetectorCascade() -{ - release(); - - delete foregroundDetector; - delete varianceFilter; - delete ensembleClassifier; - delete nnClassifier; - delete detectionResult; - delete clustering; -} - -void DetectorCascade::init() -{ - if(imgWidth == -1 || imgHeight == -1 || imgWidthStep == -1 || objWidth == -1 || objHeight == -1) - { - //printf("Error: Window dimensions not set\n"); //TODO: Convert this to exception - } - - initWindowsAndScales(); - initWindowOffsets(); - - propagateMembers(); - - ensembleClassifier->init(); - - initialised = true; -} - -//TODO: This is error-prone. Better give components a reference to DetectorCascade? -void DetectorCascade::propagateMembers() -{ - detectionResult->init(numWindows, numTrees); - - varianceFilter->windowOffsets = windowOffsets; - ensembleClassifier->windowOffsets = windowOffsets; - ensembleClassifier->imgWidthStep = imgWidthStep; - ensembleClassifier->numScales = numScales; - ensembleClassifier->scales = scales; - ensembleClassifier->numFeatures = numFeatures; - ensembleClassifier->numTrees = numTrees; - nnClassifier->windows = windows; - clustering->windows = windows; - clustering->numWindows = numWindows; - - foregroundDetector->minBlobSize = minSize * minSize; - - foregroundDetector->detectionResult = detectionResult; - varianceFilter->detectionResult = detectionResult; - ensembleClassifier->detectionResult = detectionResult; - nnClassifier->detectionResult = detectionResult; - clustering->detectionResult = detectionResult; -} - -void DetectorCascade::release() -{ - if(!initialised) - { - return; //Do nothing - } - - initialised = false; - - foregroundDetector->release(); - ensembleClassifier->release(); - nnClassifier->release(); +#include - clustering->release(); - numWindows = 0; - numScales = 0; - delete[] scales; - scales = NULL; - delete[] windows; - windows = NULL; - delete[] windowOffsets; - windowOffsets = NULL; +#include +#include +#include +#include +#define SUCCESS 0 +#define FAILURE 1 +using namespace std; - objWidth = -1; - objHeight = -1; - - detectionResult->release(); -} - -void DetectorCascade::cleanPreviousData() -{ - detectionResult->reset(); -} - -/* returns number of bounding boxes, bounding boxes, number of scales, scales - * bounding boxes are stored in an array of size 5*numBBs using the format - * scales are stored using the format - * - */ -void DetectorCascade::initWindowsAndScales() -{ - - int scanAreaX = 1; // It is important to start with 1/1, because the integral images aren't defined at pos(-1,-1) due to speed reasons - int scanAreaY = 1; - int scanAreaW = imgWidth - 1; - int scanAreaH = imgHeight - 1; - - int windowIndex = 0; - - scales = new Size[maxScale - minScale + 1]; - - numWindows = 0; - - int scaleIndex = 0; - - for(int i = minScale; i <= maxScale; i++) - { - float scale = pow(1.2, i); - int w = (int)objWidth * scale; - int h = (int)objHeight * scale; - int ssw, ssh; - - if(useShift) - { - ssw = max(1, w * shift); - ssh = max(1, h * shift); - } - else - { - ssw = 1; - ssh = 1; - } - - if(w < minSize || h < minSize || w > scanAreaW || h > scanAreaH) continue; - - scales[scaleIndex].width = w; - scales[scaleIndex].height = h; - - scaleIndex++; - - numWindows += floor((float)(scanAreaW - w + ssw) / ssw) * floor((float)(scanAreaH - h + ssh) / ssh); - } - - numScales = scaleIndex; - - windows = new int[TLD_WINDOW_SIZE * numWindows]; - - for(scaleIndex = 0; scaleIndex < numScales; scaleIndex++) - { - int w = scales[scaleIndex].width; - int h = scales[scaleIndex].height; - - int ssw, ssh; - - if(useShift) - { - ssw = max(1, w * shift); - ssh = max(1, h * shift); - } - else - { - ssw = 1; - ssh = 1; - } - - for(int y = scanAreaY; y + h <= scanAreaY + scanAreaH; y += ssh) - { - for(int x = scanAreaX; x + w <= scanAreaX + scanAreaW; x += ssw) - { - int *bb = &windows[TLD_WINDOW_SIZE * windowIndex]; - tldCopyBoundaryToArray(x, y, w, h, bb); - bb[4] = scaleIndex; - - windowIndex++; - } - } - - } - - assert(windowIndex == numWindows); -} +using namespace cv; -//Creates offsets that can be added to bounding boxes -//offsets are contained in the form delta11, delta12,... (combined index of dw and dh) -//Order: scale->tree->feature -void DetectorCascade::initWindowOffsets() +namespace tld { - windowOffsets = new int[TLD_WINDOW_OFFSET_SIZE * numWindows]; - int *off = windowOffsets; - - int windowSize = TLD_WINDOW_SIZE; + //TODO: Convert this to a function +#define sub2idx(x,y,imgWidthStep) ((int) (floor((x)+0.5) + floor((y)+0.5)*(imgWidthStep))) - for(int i = 0; i < numWindows; i++) - { + DetectorCascade::DetectorCascade() + { + objWidth = -1; //MUST be set before calling init + objHeight = -1; //MUST be set before calling init + useShift = 1; + imgHeight = -1; + imgWidth = -1; + + shift = 0.1; + minScale = -10; + maxScale = 10; + minSize = 25; + imgWidthStep = -1; + + numTrees = 10; + numFeatures = 13; + numFrame = 0; + + initialised = false; + oclSetup(); + char * kernelName = "..\\..\\..\\src\\libopentld\\kernel\\EnsembleClassifierFilter_kernel.cl.cpp"; + oclBuildKernel(kernelName); + + + foregroundDetector = new ForegroundDetector(); + varianceFilter = new VarianceFilter(); + ensembleClassifier = new EnsembleClassifier(); + nnClassifier = new NNClassifier(); + clustering = new Clustering(); + + detectionResult = new DetectionResult(); + } + + DetectorCascade::~DetectorCascade() + { + release(); + + delete foregroundDetector; + delete varianceFilter; + delete ensembleClassifier; + delete nnClassifier; + delete detectionResult; + delete clustering; + } + + + int DetectorCascade::convertToString(const char *filename, std::string& s) + { + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); + + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } + + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return false; + } + + void DetectorCascade::init() + { + if (imgWidth == -1 || imgHeight == -1 || imgWidthStep == -1 || objWidth == -1 || objHeight == -1) + { + //printf("Error: Window dimensions not set\n"); //TODO: Convert this to exception + } + + initWindowsAndScales(); + initWindowOffsets(); + + propagateMembers(); + + ensembleClassifier->init(); + + initialised = true; + } + + //TODO: This is error-prone. Better give components a reference to DetectorCascade? + void DetectorCascade::propagateMembers() + { + detectionResult->init(numWindows, numTrees); + + varianceFilter->windowOffsets = windowOffsets; + varianceFilter->numWindows = numWindows; + ensembleClassifier->numWindows = numWindows;//added by xqc + ensembleClassifier->windowOffsets = windowOffsets; + ensembleClassifier->imgWidthStep = imgWidthStep; + ensembleClassifier->numScales = numScales; + ensembleClassifier->scales = scales; + ensembleClassifier->numFeatures = numFeatures; + ensembleClassifier->numTrees = numTrees; + nnClassifier->windows = windows; + nnClassifier->numWindows = numWindows; + clustering->windows = windows; + clustering->numWindows = numWindows; + + foregroundDetector->minBlobSize = minSize * minSize; + + foregroundDetector->detectionResult = detectionResult; + varianceFilter->detectionResult = detectionResult; + ensembleClassifier->detectionResult = detectionResult; + ensembleClassifier->varianceFilter = varianceFilter; + nnClassifier->detectionResult = detectionResult; + clustering->detectionResult = detectionResult; + + + + //ocl + varianceFilter->platform = platform; + varianceFilter->devices = devices; + varianceFilter->context = context; + varianceFilter->commandQueue = commandQueue; + varianceFilter->program = program; + varianceFilter->kernel_intgegral_cols = kernel_intgegral_cols; + + varianceFilter->kernel_intgegral_rows = kernel_intgegral_rows; + + + // + ensembleClassifier->platform = platform; + ensembleClassifier->devices = devices; + ensembleClassifier->context = context; + ensembleClassifier->commandQueue = commandQueue; + ensembleClassifier->program = program; + ensembleClassifier->variance_ensemble_kernel = kernel_ensemble; + ensembleClassifier->kernel_intgegral_cols_en = kernel_intgegral_cols; + + ensembleClassifier->kernel_intgegral_rows_en = kernel_intgegral_rows; + //kernel_variance = clCreateKernel(program, "varianceFilter", NULL); + + //nnClassifier + nnClassifier->platform = platform; + nnClassifier->devices = devices; + nnClassifier->context = context; + nnClassifier->commandQueue = commandQueue; + nnClassifier->program = program; + nnClassifier->kernel_nnClassifier = kernel_nnClassifier; + + + + } + + void DetectorCascade::release() + { + if (!initialised) + { + return; //Do nothing + } + + initialised = false; + + foregroundDetector->release(); + ensembleClassifier->release(); + nnClassifier->release(); + + clustering->release(); + + numWindows = 0; + numScales = 0; + + delete[] scales; + scales = NULL; + delete[] windows; + windows = NULL; + delete[] windowOffsets; + windowOffsets = NULL; + + objWidth = -1; + objHeight = -1; + + detectionResult->release(); + oclRelease(); + } + + void DetectorCascade::cleanPreviousData() + { + detectionResult->reset(); + } + + /* returns number of bounding boxes, bounding boxes, number of scales, scales + * bounding boxes are stored in an array of size 5*numBBs using the format + * scales are stored using the format + * + */ + void DetectorCascade::initWindowsAndScales() + { + + int scanAreaX = 1; // It is important to start with 1/1, because the integral images aren't defined at pos(-1,-1) due to speed reasons + int scanAreaY = 1; + int scanAreaW = imgWidth - 1; + int scanAreaH = imgHeight - 1; + + int windowIndex = 0; + + scales = new Size[maxScale - minScale + 1]; + + numWindows = 0; + + int scaleIndex = 0; + + for (int i = minScale; i <= maxScale; i++) + { + float scale = pow(1.2, i); + int w = (int)objWidth * scale; + int h = (int)objHeight * scale; + int ssw, ssh; + + if (useShift) + { + ssw = max(1, w * shift); + ssh = max(1, h * shift); + } + else + { + ssw = 1; + ssh = 1; + } + + if (w < minSize || h < minSize || w > scanAreaW || h > scanAreaH) continue; + + scales[scaleIndex].width = w; + scales[scaleIndex].height = h; + + scaleIndex++; + + numWindows += floor((float)(scanAreaW - w + ssw) / ssw) * floor((float)(scanAreaH - h + ssh) / ssh); + } + + numScales = scaleIndex; + + windows = new int[TLD_WINDOW_SIZE * numWindows]; + + for (scaleIndex = 0; scaleIndex < numScales; scaleIndex++) + { + int w = scales[scaleIndex].width; + int h = scales[scaleIndex].height; + + int ssw, ssh; + + if (useShift) + { + ssw = max(1, w * shift); + ssh = max(1, h * shift); + } + else + { + ssw = 1; + ssh = 1; + } + + for (int y = scanAreaY; y + h <= scanAreaY + scanAreaH; y += ssh) + { + for (int x = scanAreaX; x + w <= scanAreaX + scanAreaW; x += ssw) + { + int *bb = &windows[TLD_WINDOW_SIZE * windowIndex]; + tldCopyBoundaryToArray(x, y, w, h, bb); + bb[4] = scaleIndex; + + windowIndex++; + } + } + + } + + assert(windowIndex == numWindows); + printf("******** DetectorCascade::initWindowsAndScales()********************%d**********************************\n", numWindows); + } + + //oclSetup + int DetectorCascade::oclSetup() + { + /*Step1: Getting platforms and choose an available one.*/ + + platform = NULL; //the chosen platform + status = clGetPlatformIDs(0, NULL, &numPlatforms); + if (status != CL_SUCCESS) + { + cout << "DetectorCascade oclSetup Error: Getting platforms!" << endl; + return 0; + } + + /*For clarity, choose the first available platform. */ + if (numPlatforms > 0) + { + cl_platform_id* platforms = (cl_platform_id*)malloc(numPlatforms* sizeof(cl_platform_id)); + status = clGetPlatformIDs(numPlatforms, platforms, NULL); + platform = platforms[0]; + free(platforms); + } + /*Step 2:Query the platform and choose the first GPU device if has one.Otherwise use the CPU as device.*/ + numDevices = 0; + + status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 0, NULL, &numDevices); + cout << "numDevices::" << numDevices << endl; + if (numDevices == 0) //no GPU available. + { + cout << "No GPU device available." << endl; + cout << "Choose CPU as default device." << endl; + status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_CPU, 0, NULL, &numDevices); + devices = (cl_device_id*)malloc(numDevices * sizeof(cl_device_id)); + status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_CPU, numDevices, devices, NULL); + } + else + { + devices = (cl_device_id*)malloc(numDevices * sizeof(cl_device_id)); + status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, numDevices, devices, NULL); + } + for (cl_uint i = 0; i < numDevices; ++i) + { + char deviceName[1024]; + status = clGetDeviceInfo(devices[i], CL_DEVICE_NAME, sizeof(deviceName), + deviceName, NULL); + + std::cout << "Device " << i << " : " << deviceName + << " Device ID is " << devices[i] << std::endl; + } + + /*Step 3: Create context.*/ + context = clCreateContext(NULL, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + + /*Step 4: Creating command queue associate with the context.*/ + commandQueue = clCreateCommandQueue(context, devices[SELECTED_DEVICE_ID], 0, NULL); + } + int DetectorCascade::oclBuildKernel(const char *kernelName) + { + /*Step 5: Create program object */ + //const char *filename = "M:\\OpenTLD\\OpenTLD-master\\OpenTLD-master\\src\\opentld\\HelloWorld_Kernel.cl"; + + + string sourceStr; + status = convertToString(kernelName, sourceStr); + const char *source = sourceStr.c_str(); + size_t sourceSize[] = { strlen(source) }; + program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); + + /*Step 6: Build program. */ + status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + if (status != CL_SUCCESS) + { + size_t log_size; + char* program_log; + clGetProgramBuildInfo(program, devices[SELECTED_DEVICE_ID], CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size); + program_log = (char*)malloc(log_size + 1); + clGetProgramBuildInfo(program, devices[SELECTED_DEVICE_ID], CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL); + printf("%s\n", program_log); + free(program_log); + + + cout << "DetectorCascade::oclBuildKernelError:clBuildProgram !" << endl; + return false; + } + + kernel_variance = clCreateKernel(program, "varianceFilter", NULL); + kernel_ensemble = clCreateKernel(program, "EnsembleClassifierFilter", NULL); + kernel_intgegral_cols = clCreateKernel(program, "integral_cols_DD", NULL); + kernel_intgegral_rows = clCreateKernel(program, "integral_rows_DD", NULL); + kernel_nnClassifier = clCreateKernel(program, "nnClassifier", NULL); + } + + void DetectorCascade::oclRelease() + { + /*Step 12: Clean the resources.*/ + if (kernel != NULL) + status = clReleaseKernel(kernel); //Release kernel. + //status = clReleaseProgram(program); //Release the program object. + //status = clReleaseMemObject(inputBuffer); //Release mem object. + //status = clReleaseMemObject(outputBuffer); + //status = clReleaseCommandQueue(commandQueue); //Release Command queue. + //status = clReleaseContext(context); //Release context. + if (devices != NULL) + { + free(devices); + devices = NULL; + } + } + + //Creates offsets that can be added to bounding boxes + //offsets are contained in the form delta11, delta12,... (combined index of dw and dh) + //Order: scale->tree->feature + void DetectorCascade::initWindowOffsets() + { + + windowOffsets = new int[TLD_WINDOW_OFFSET_SIZE * numWindows]; + + int *off = windowOffsets; + + int windowSize = TLD_WINDOW_SIZE; + //#pragma omp parallel for num_threads(8) + for (int i = 0; i < numWindows; i++) + { + int *window = windows + windowSize * i; + *off++ = sub2idx(window[0] - 1, window[1] - 1, imgWidthStep); // x1-1,y1-1 + *off++ = sub2idx(window[0] - 1, window[1] + window[3] - 1, imgWidthStep); // x1-1,y2 + *off++ = sub2idx(window[0] + window[2] - 1, window[1] - 1, imgWidthStep); // x2,y1-1 + *off++ = sub2idx(window[0] + window[2] - 1, window[1] + window[3] - 1, imgWidthStep); // x2,y2 + *off++ = window[4] * 2 * numFeatures * numTrees; // pointer to features for this scale + *off++ = window[2] * window[3]; //Area of bounding box + } + } + + + void DetectorCascade::cldetect(const Mat &img) + { + //For every bounding box, the output is confidence, pattern, variance + //numFrame++; printf("the num of the frame id is %d,numWindows=%d\n", numFrame, numWindows); + + detectionResult->reset(); + if (!initialised) + { + return; + } + + //Prepare components + foregroundDetector->nextIteration(img); //Calculates foreground + //double tic = cvGetTickCount(); + //varianceFilter->nextIteration(img); //Calculates integral images + //varianceFilter->filter2(); + + //double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + //toc = toc / 1000000; + //printf("*****the integral time used is %f************\n", toc);//float fps = 1 / toc; + ensembleClassifier->nextIteration(img); + + + //varianceFilter->clfilter(img); + ensembleClassifier->clfilter(img); + + //for (int i = 0; i < numWindows; i++) + //{ + //printf(" && detectionResult->windowFlags[%d]=%d\n", i, detectionResult->windowFlags[i]); + /* if (detectionResult->windowFlags[i] == 0) + { + printf("xxxxxxxxxxxxxxx windowIdx = %d\nxxxxxxxxxxxxxxxxxx", i); + Sleep(500); + }*/ + + //} + //for (int i = 0; i < numWindows; i++) + //{ + // //printf("flag=1 windowsidx = %d,flag=%d\n", i, detectionResult->windowFlags[i]); + // if (detectionResult->variances != 0) + // { + // printf("haha ... windowsidx = %d, detectionResult->variances=%f\n", i, detectionResult->variances[i]); + // Sleep(1000); + // } + //} + + //printf("detectionResult[753666]=%f\n",detectionResult->posteriors[753666]); +#if clNNClassifier + for (int i = 0, count = 0; i < numWindows; i++) + { + if (detectionResult->posteriors[i] >= 0.5f) + { + //float values; + //int index; + //bool flag; + nnClassifier->nnClassifyStructInstance.conf = 0.0f; + nnClassifier->nnClassifyStructInstance.index = i; + nnClassifier->nnClassifyStructInstance.flag = false; + nnClassifier->candidatesToNNClassifyVector->push_back(nnClassifier->nnClassifyStructInstance); + + nnClassifier->candidatesToNNClassifyIndexVector->push_back(i); + } + + } + if (nnClassifier->candidatesToNNClassifyIndexVector->size()==0) + goto EndNNclassify; + + + nnClassifier->clNNFilter(img); + + + + + for (int i = 0; i < nnClassifier->candidatesToNNClassifyVector->size(); i++) + { + if (nnClassifier->candidatesToNNClassifyVector->at(i).conf > nnClassifier->thetaTP) + detectionResult->confidentIndices->push_back(nnClassifier->candidatesToNNClassifyVector->at(i).index); + } + + + + + + + + //返回 + nnClassifier->candidatesToNNClassifyIndexVector->clear(); + + nnClassifier->candidatesToNNClassifyVector->clear(); + + +#else + for (int i = 0, count = 0; i < numWindows; i++) + { + // printf("I am victor %d\n", i); + //if (detectionResult->windowFlags[i] == 0) + if (detectionResult->posteriors[i] >= 0.5f) + { + // printf("windowIDx= %d,count=%d\n", i, count++); + //count++; + //printf("Framenum = %d, I am victor %d,count=%d\n", FramNum, i, count); + if (!nnClassifier->filter(img, i)) + { + continue; + } + //count++; + + detectionResult->confidentIndices->push_back(i); + + } + + } + + +#endif + + + + //printf("confident_size %d\n\n", detectionResult->confidentIndices->size());//zhaodc + + //Cluster + EndNNclassify: + clustering->clusterConfidentIndices(); + + detectionResult->containsValidData = true; + + EndofCldetect: + ; - int *window = windows + windowSize * i; - *off++ = sub2idx(window[0] - 1, window[1] - 1, imgWidthStep); // x1-1,y1-1 - *off++ = sub2idx(window[0] - 1, window[1] + window[3] - 1, imgWidthStep); // x1-1,y2 - *off++ = sub2idx(window[0] + window[2] - 1, window[1] - 1, imgWidthStep); // x2,y1-1 - *off++ = sub2idx(window[0] + window[2] - 1, window[1] + window[3] - 1, imgWidthStep); // x2,y2 - *off++ = window[4] * 2 * numFeatures * numTrees; // pointer to features for this scale - *off++ = window[2] * window[3]; //Area of bounding box - } } + void DetectorCascade::detect(const Mat &img) + { + //For every bounding box, the output is confidence, pattern, variance -void DetectorCascade::detect(const Mat &img) -{ - //For every bounding box, the output is confidence, pattern, variance - - detectionResult->reset(); + detectionResult->reset(); - if(!initialised) - { - return; - } + if (!initialised) + { + return; + } - //Prepare components - foregroundDetector->nextIteration(img); //Calculates foreground - varianceFilter->nextIteration(img); //Calculates integral images - ensembleClassifier->nextIteration(img); + //Prepare components + foregroundDetector->nextIteration(img); //Calculates foreground + varianceFilter->nextIteration(img); //Calculates integral images + ensembleClassifier->nextIteration(img); - #pragma omp parallel for +#pragma omp parallel for - for(int i = 0; i < numWindows; i++) - { + for (int i = 0; i < numWindows; i++) + { - int *window = &windows[TLD_WINDOW_SIZE * i]; + int *window = &windows[TLD_WINDOW_SIZE * i]; - if(foregroundDetector->isActive()) - { - bool isInside = false; + if (foregroundDetector->isActive()) + { + bool isInside = false; - for(size_t j = 0; j < detectionResult->fgList->size(); j++) - { + for (size_t j = 0; j < detectionResult->fgList->size(); j++) + { - int bgBox[4]; - tldRectToArray(detectionResult->fgList->at(j), bgBox); + int bgBox[4]; + tldRectToArray(detectionResult->fgList->at(j), bgBox); - if(tldIsInside(window, bgBox)) //TODO: This is inefficient and should be replaced by a quadtree - { - isInside = true; - } - } + if (tldIsInside(window, bgBox)) //TODO: This is inefficient and should be replaced by a quadtree + { + isInside = true; + } + } - if(!isInside) - { - detectionResult->posteriors[i] = 0; - continue; - } - } + if (!isInside) + { + detectionResult->posteriors[i] = 0; + continue; + } + } - if(!varianceFilter->filter(i)) - { - detectionResult->posteriors[i] = 0; - continue; - } + if (!varianceFilter->filter(i)) + { + detectionResult->posteriors[i] = 0; + continue; + } - if(!ensembleClassifier->filter(i)) - { - continue; - } + if (!ensembleClassifier->filter(i)) + { + continue; + } - if(!nnClassifier->filter(img, i)) - { - continue; - } + if (!nnClassifier->filter(img, i)) + { + continue; + } - detectionResult->confidentIndices->push_back(i); + detectionResult->confidentIndices->push_back(i); - } + } - //Cluster - clustering->clusterConfidentIndices(); + //Cluster + clustering->clusterConfidentIndices(); - detectionResult->containsValidData = true; -} + detectionResult->containsValidData = true; + } } /* namespace tld */ diff --git a/src/libopentld/tld/DetectorCascade.h b/src/libopentld/tld/DetectorCascade.h index cbad97e3..d8fa8242 100644 --- a/src/libopentld/tld/DetectorCascade.h +++ b/src/libopentld/tld/DetectorCascade.h @@ -17,82 +17,122 @@ * */ /* - * DetectorCascade.h - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* DetectorCascade.h +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #ifndef DETECTORCASCADE_H_ #define DETECTORCASCADE_H_ + #include "DetectionResult.h" #include "ForegroundDetector.h" #include "VarianceFilter.h" #include "EnsembleClassifier.h" #include "Clustering.h" #include "NNClassifier.h" - +#include +#include "switch.h" +#define TRUE 0 +#define FALSE 1 namespace tld { -//Constants -static const int TLD_WINDOW_SIZE = 5; -static const int TLD_WINDOW_OFFSET_SIZE = 6; + //Constants + static const int TLD_WINDOW_SIZE = 5; + static const int TLD_WINDOW_OFFSET_SIZE = 6; -class DetectorCascade -{ - //Working data - int numScales; - cv::Size *scales; -public: - //Configurable members - int minScale; - int maxScale; - bool useShift; - float shift; - int minSize; - int numFeatures; - int numTrees; - - //Needed for init - int imgWidth; - int imgHeight; - int imgWidthStep; - int objWidth; - int objHeight; - - int numWindows; - int *windows; - int *windowOffsets; - - //State data - bool initialised; - - //Components of Detector Cascade - ForegroundDetector *foregroundDetector; - VarianceFilter *varianceFilter; - EnsembleClassifier *ensembleClassifier; - Clustering *clustering; - NNClassifier *nnClassifier; - - DetectionResult *detectionResult; - - void propagateMembers(); - - DetectorCascade(); - ~DetectorCascade(); - - void init(); - - void initWindowOffsets(); - void initWindowsAndScales(); - - void release(); - void cleanPreviousData(); - void detect(const cv::Mat &img); -}; + class DetectorCascade + { + //Working data + int numScales; + cv::Size *scales; + + public: + //Configurable members + int numFrame; + int minScale; + int maxScale; + bool useShift; + float shift; + int minSize; + int numFeatures; + int numTrees; + + //Needed for init + int imgWidth; + int imgHeight; + int imgWidthStep; + int objWidth; + int objHeight; + int FramNum; + int numWindows; + int *windows; + int *windowOffsets; + + + + //******ocl enviroments + + cl_uint numPlatforms; //the NO. of platforms + cl_platform_id platform; //the chosen platform + cl_int status; + cl_uint numDevices; + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel kernel; + + cl_kernel kernel_ensemble; + cl_kernel kernel_variance; + cl_kernel kernel_intgegral_cols; + cl_kernel kernel_intgegral_rows; + cl_kernel kernel_nnClassifier; + + int oclSetup(); + int oclBuildKernel(const char *kernelName); + + + void oclRelease(); + + //******ocl enviroments end + //State data + bool initialised; + + //Components of Detector Cascade + ForegroundDetector *foregroundDetector; + VarianceFilter *varianceFilter; + EnsembleClassifier *ensembleClassifier; + Clustering *clustering; + NNClassifier *nnClassifier; + + DetectionResult *detectionResult; + + + + DetectorCascade(); + ~DetectorCascade(); + + void init(); + + void initWindowOffsets(); + void initWindowsAndScales(); + + void release(); + void cleanPreviousData(); + + void cldetect(const cv::Mat &img); + int convertToString(const char *filename, std::string& s); + + void propagateMembers(); + void detect(const cv::Mat &img); + }; } /* namespace tld */ #endif /* DETECTORCASCADE_H_ */ diff --git a/src/libopentld/tld/EnsembleClassifier.cpp b/src/libopentld/tld/EnsembleClassifier.cpp index 8c43cc05..d9c8dc4c 100644 --- a/src/libopentld/tld/EnsembleClassifier.cpp +++ b/src/libopentld/tld/EnsembleClassifier.cpp @@ -17,13 +17,14 @@ * */ /* - * EnsembleClassifier.cpp - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* EnsembleClassifier.cpp +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #include "DetectorCascade.h" +#include #include #include @@ -31,221 +32,729 @@ #include #include "EnsembleClassifier.h" - +#include +#include using namespace std; using namespace cv; +#include +#include + +#define ROUND_K(x,k) ((x+(k-1))/(k))*(k) namespace tld { -//TODO: Convert this to a function + //TODO: Convert this to a function #define sub2idx(x,y,widthstep) ((int) (floor((x)+0.5) + floor((y)+0.5)*(widthstep))) -EnsembleClassifier::EnsembleClassifier() : - features(NULL), - featureOffsets(NULL), - posteriors(NULL), - positives(NULL), - negatives(NULL) -{ - numTrees = 10; - numFeatures = 13; - enabled = true; -} - -EnsembleClassifier::~EnsembleClassifier() -{ - release(); -} - - -void EnsembleClassifier::init() -{ - numIndices = pow(2.0f, numFeatures); - - initFeatureLocations(); - initFeatureOffsets(); - initPosteriors(); -} - -void EnsembleClassifier::release() -{ - delete[] features; - features = NULL; - delete[] featureOffsets; - featureOffsets = NULL; - delete[] posteriors; - posteriors = NULL; - delete[] positives; - positives = NULL; - delete[] negatives; - negatives = NULL; -} - -/* - * Generates random measurements in the format - */ -void EnsembleClassifier::initFeatureLocations() -{ - int size = 2 * 2 * numFeatures * numTrees; - - features = new float[size]; + EnsembleClassifier::EnsembleClassifier() : + features(NULL), + featureOffsets(NULL), + posteriors(NULL), + positives(NULL), + negatives(NULL) + { + numTrees = 10; + numFeatures = 13; + enabled = true; - for(int i = 0; i < size; i++) - { - features[i] = rand() / (float)RAND_MAX; - } + } -} + EnsembleClassifier::~EnsembleClassifier() + { + release(); + } -//Creates offsets that can be added to bounding boxes -//offsets are contained in the form delta11, delta12,... (combined index of dw and dh) -//Order: scale.tree->feature -void EnsembleClassifier::initFeatureOffsets() -{ - featureOffsets = new int[numScales * numTrees * numFeatures * 2]; - int *off = featureOffsets; + void EnsembleClassifier::init() + { + numIndices = pow(2.0f, numFeatures); - for(int k = 0; k < numScales; k++) - { - Size scale = scales[k]; + initFeatureLocations(); + initFeatureOffsets(); + initPosteriors(); + int tld_window_offset_size = TLD_WINDOW_OFFSET_SIZE; - for(int i = 0; i < numTrees; i++) - { - for(int j = 0; j < numFeatures; j++) - { - float *currentFeature = features + (4 * numFeatures) * i + 4 * j; - *off++ = sub2idx((scale.width - 1) * currentFeature[0] + 1, (scale.height - 1) * currentFeature[1] + 1, imgWidthStep); //We add +1 because the index of the bounding box points to x-1, y-1 - *off++ = sub2idx((scale.width - 1) * currentFeature[2] + 1, (scale.height - 1) * currentFeature[3] + 1, imgWidthStep); - } - } - } -} + //oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), (void *)windowOffsets, NULL); + //oclbufffeatureOffsets = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (numScales * numTrees * numFeatures * 2) * sizeof(int), (void *)featureOffsets, NULL); + //oclbuffDetectionResultfeatureVectors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows * numTrees)* sizeof(int), (void *)detectionResult->featureVectors, NULL); + //oclbuffDetectionResultPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, NULL); + // + //oclbuffPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numTrees * numIndices)* sizeof(float), (void *)posteriors, NULL); + //oclbuffDetectionResultVarious = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->variances, NULL); -void EnsembleClassifier::initPosteriors() -{ - posteriors = new float[numTrees * numIndices]; - positives = new int[numTrees * numIndices]; - negatives = new int[numTrees * numIndices]; - - for(int i = 0; i < numTrees; i++) - { - for(int j = 0; j < numIndices; j++) - { - posteriors[i * numIndices + j] = 0; - positives[i * numIndices + j] = 0; - negatives[i * numIndices + j] = 0; - } - } -} - -void EnsembleClassifier::nextIteration(const Mat &img) -{ - if(!enabled) return; - this->img = (const unsigned char *)img.data; -} -//Classical fern algorithm -int EnsembleClassifier::calcFernFeature(int windowIdx, int treeIdx) -{ - int index = 0; - int *bbox = windowOffsets + windowIdx * TLD_WINDOW_OFFSET_SIZE; - int *off = featureOffsets + bbox[4] + treeIdx * 2 * numFeatures; //bbox[4] is pointer to features for the current scale - for(int i = 0; i < numFeatures; i++) - { - index <<= 1; - int fp0 = img[bbox[0] + off[0]]; - int fp1 = img[bbox[0] + off[1]]; - if(fp0 > fp1) - { - index |= 1; - } + } + + void EnsembleClassifier::release() + { + delete[] features; + features = NULL; + delete[] featureOffsets; + featureOffsets = NULL; + delete[] posteriors; + posteriors = NULL; + delete[] positives; + positives = NULL; + delete[] negatives; + negatives = NULL; + + + + clReleaseKernel(variance_ensemble_kernel); + clReleaseProgram(program); + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbufffeatureOffsets); + clReleaseMemObject(oclbuffDetectionResultfeatureVectors); + clReleaseMemObject(oclbuffDetectionResultPosteriors); + clReleaseMemObject(oclbuffPosteriors); + clReleaseMemObject(oclbuffDetectionResultVarious); + + + } + + /* + * Generates random measurements in the format + */ + void EnsembleClassifier::initFeatureLocations() + { + int size = 2 * 2 * numFeatures * numTrees; + + features = new float[size]; + + for (int i = 0; i < size; i++) + { + features[i] = rand() / (float)RAND_MAX; + } + + } + + //Creates offsets that can be added to bounding boxes + //offsets are contained in the form delta11, delta12,... (combined index of dw and dh) + //Order: scale.tree->feature + void EnsembleClassifier::initFeatureOffsets() + { + + featureOffsets = new int[numScales * numTrees * numFeatures * 2]; + int *off = featureOffsets; + long int offsum1, offsum2; + + for (int k = 0; k < numScales; k++) + { + Size scale = scales[k]; + + for (int i = 0; i < numTrees; i++) + { + for (int j = 0; j < numFeatures; j++) + { + + float *currentFeature = features + (4 * numFeatures) * i + 4 * j; + offsum1 = sub2idx((scale.width - 1) * currentFeature[0] + 1, (scale.height - 1) * currentFeature[1] + 1, imgWidthStep); //We add +1 because the index of the bounding box points to x-1, y-1 + *off++ = offsum1; + + offsum2 = sub2idx((scale.width - 1) * currentFeature[2] + 1, (scale.height - 1) * currentFeature[3] + 1, imgWidthStep); + *off++ = offsum2; + } + } + } + } + + + int EnsembleClassifier::convertToString(const char *filename, std::string& s) + { + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); + + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } + + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return false; + } + void EnsembleClassifier::initPosteriors() + { + posteriors = new float[numTrees * numIndices]; + positives = new int[numTrees * numIndices]; + negatives = new int[numTrees * numIndices]; + + for (int i = 0; i < numTrees; i++) + { + for (int j = 0; j < numIndices; j++) + { + posteriors[i * numIndices + j] = 0; + positives[i * numIndices + j] = 0; + negatives[i * numIndices + j] = 0; + } + } + } + + + + void EnsembleClassifier::integralImag_extract(const Mat &src, Mat &sum, Mat &sqsum)//void VarianceFilter::integralImag_extract(const Mat &src) + { + + + int i, j; + + cv::Mat t_sum; + //cv::Mat sum; + cv::Mat t_sqsum; + //cv::Mat sqsum; + + int vlen = 4; cl_event events[2]; + int offset = 0; + int pre_invalid = 0; + int vcols = (pre_invalid + src.cols + vlen - 1) / vlen; + + int w = src.cols, h = src.rows; + int depth = src.depth() == CV_8U ? CV_32S : CV_64F; + int type = CV_MAKE_TYPE(depth, 1); + +#ifdef debug + printf("the type is %d\n", depth); +#endif + t_sum.create(src.cols, src.rows, type); + sum.create(h, w, type); + + t_sqsum.create(src.cols, src.rows, CV_32FC1); + sqsum.create(h, w, CV_32FC1); + + int sum_offset = 0; // sum.offset / vlen; + int sqsum_offset = 0; // sqsum.offset / vlen; + + + cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY, (src.cols) * (src.rows) * sizeof(uchar), NULL, NULL); + cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(int), NULL, NULL); + cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(float), NULL, NULL); + cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(int), NULL, NULL); + cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(float), NULL, NULL); + //use host ptr + //cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (src.cols) * (src.rows) * sizeof(uchar), (void *)src.data, NULL); + //cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)t_sum.data, NULL); + //cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR , (src.cols) * (src.rows) * sizeof(float), (void *)t_sqsum.data, NULL); + //cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)sum.data, NULL); + //cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(float), (void *)sqsum.data, NULL); + + int step; + int src_step, sum_step; + int t_sum_step, sqsum_step; + + //src_step = 1920; + //t_sum_step = 4320; + //sum_step = 7680; + //sqsum_step = 7680; + int rows = src.rows; + int cols = src.cols; + src_step = src.step; + t_sum_step = t_sum.step; + sum_step = sum.step; + sqsum_step = sqsum.step; + + //printf(""); + status = clEnqueueWriteBuffer(commandQueue, srcdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(uchar), src.data, 0, NULL, NULL); + + assert(status == CL_SUCCESS); + //if (status != CL_SUCCESS) + //{ + // cout << "Error: src.data kernel_intgegral_cols_en EnqueueNDRangeKernel!" << endl; + //} + + status = clSetKernelArg(kernel_intgegral_cols_en, 0, sizeof(cl_mem), (void *)&srcdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&src.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 1, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 2, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 3, sizeof(cl_int), (void *)&offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&offset)); + status = clSetKernelArg(kernel_intgegral_cols_en, 4, sizeof(cl_int), (void *)&pre_invalid);//args.push_back(make_pair(sizeof(cl_int), (void *)&pre_invalid)); + status = clSetKernelArg(kernel_intgegral_cols_en, 5, sizeof(cl_int), (void *)&rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.rows)); + status = clSetKernelArg(kernel_intgegral_cols_en, 6, sizeof(cl_int), (void *)&cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.cols)); + status = clSetKernelArg(kernel_intgegral_cols_en, 7, sizeof(cl_int), &src_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.step)); + status = clSetKernelArg(kernel_intgegral_cols_en, 8, sizeof(cl_int), &t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + + + + size_t gt[3] = { ((vcols + 1) / 2) * 256, 1, 1 }, lt[3] = { 256, 1, 1 }; + // //openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_cols", gt, lt, args, -1, depth); + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols_en, 1, NULL, gt, lt, 0, NULL, &events[0]); + status = clWaitForEvents(1, &events[0]); + assert(status == CL_SUCCESS); + // + // status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols_en, 1, NULL, gt, lt, 0, NULL, NULL); + // + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_cols_en EnqueueNDRangeKernel!" << endl; + //} + // status = clWaitForEvents(1, &events[0]); + + //if (status != CL_SUCCESS) + // { + // printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + // } + + /*status = clEnqueueReadBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueReadBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueWriteBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueWriteBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + */ + + //clReleaseMemObject(tsumdata); + //clReleaseMemObject(tsqsumdata); + + + // tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)t_sum.data, NULL); + // tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(float), (void *)t_sqsum.data, NULL); + + + + status = clSetKernelArg(kernel_intgegral_rows_en, 0, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 1, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 2, sizeof(cl_mem), (void *)&sumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 3, sizeof(cl_mem), (void *)&sqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 4, sizeof(cl_int), (void *)&t_sum.rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.rows)); + status = clSetKernelArg(kernel_intgegral_rows_en, 5, sizeof(cl_int), (void *)&t_sum.cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.cols)); + status = clSetKernelArg(kernel_intgegral_rows_en, 6, sizeof(cl_int), (void *)&t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 7, sizeof(cl_int), (void *)&sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 8, sizeof(cl_int), (void *)&sqsum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 9, sizeof(cl_int), (void *)&sum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum_offset)); + status = clSetKernelArg(kernel_intgegral_rows_en, 10, sizeof(cl_int), (void *)&sqsum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum_offset)); + + size_t gt2[3] = { t_sum.cols * 32, 1, 1 }, lt2[3] = { 256, 1, 1 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_rows_en, 1, NULL, gt2, lt2, 0, NULL, &events[1]);//openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_rows", gt2, lt2, args, -1, depth); + + + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_rows_en_en EnqueueNDRangeKernel!" << endl; + //} + status = clWaitForEvents(1, &events[1]); + assert(status == CL_SUCCESS); + //if (status != CL_SUCCESS) + //{ + // printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + //} + + //convert clMat to mat + status = clEnqueueReadBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueReadBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + + + //psum = (float*) &sqsum.data; + //for (i = 0; i img = (const unsigned char *)img.data; + } + + //Classical fern algorithm + int EnsembleClassifier::calcFernFeature(int windowIdx, int treeIdx) + { + + int index = 0; + int *bbox = windowOffsets + windowIdx * TLD_WINDOW_OFFSET_SIZE; + int *off = featureOffsets + bbox[4] + treeIdx * 2 * numFeatures; //bbox[4] is pointer to features for the current scale + + for (int i = 0; i < numFeatures; i++) + { + index <<= 1; + + int fp0 = img[bbox[0] + off[0]]; + int fp1 = img[bbox[0] + off[1]]; + + if (fp0 > fp1) + { + index |= 1; + } + + off += 2; + } + + return index; + } + + void EnsembleClassifier::calcFeatureVector(int windowIdx, int *featureVector) + { + for (int i = 0; i < numTrees; i++) + { + featureVector[i] = calcFernFeature(windowIdx, i); + } + } + + float EnsembleClassifier::calcConfidence(int *featureVector) + { + float conf = 0.0; + int temp = 0; + for (int i = 0; i < numTrees; i++) + { + temp = featureVector[i]; + conf += posteriors[i * numIndices + temp]; + //conf += posteriors[i * numIndices + featureVector[i]]; + } + + return conf; + } + + void EnsembleClassifier::classifyWindow2(int windowIdx) + { + float conf = 0.0; + int *featureVector = detectionResult->featureVectors + numTrees * windowIdx; + calcFeatureVector(windowIdx, featureVector); + conf = calcConfidence(featureVector); + detectionResult->posteriors[windowIdx] = conf; + + } + + bool EnsembleClassifier::filter(int i) + { + if (!enabled) return true; + + classifyWindow(i); - off += 2; - } + if (detectionResult->posteriors[i] < 0.5) return false; - return index; -} - -void EnsembleClassifier::calcFeatureVector(int windowIdx, int *featureVector) -{ - for(int i = 0; i < numTrees; i++) - { - featureVector[i] = calcFernFeature(windowIdx, i); - } -} - -float EnsembleClassifier::calcConfidence(int *featureVector) -{ - float conf = 0.0; + return true; + } + void EnsembleClassifier::classifyWindow(int windowIdx) + { + int index, *off, *bbox; + float conf = 0.0; + int treeIdx = 0; + int temp1, temp2; + int *featureVector = detectionResult->featureVectors + numTrees * windowIdx; + int idx = 0; + for (treeIdx = 0; treeIdx < numTrees; treeIdx++) + { - for(int i = 0; i < numTrees; i++) - { - conf += posteriors[i * numIndices + featureVector[i]]; - } + index = 0; - return conf; -} + bbox = windowOffsets + windowIdx * TLD_WINDOW_OFFSET_SIZE; + off = featureOffsets + bbox[4] + treeIdx * 2 * numFeatures; //bbox[4] is pointer to features for the current scale -void EnsembleClassifier::classifyWindow(int windowIdx) -{ - int *featureVector = detectionResult->featureVectors + numTrees * windowIdx; - calcFeatureVector(windowIdx, featureVector); + for (int i = 0; i < numFeatures; i++) + { + index <<= 1; - detectionResult->posteriors[windowIdx] = calcConfidence(featureVector); -} + int fp0 = img[bbox[0] + off[0]]; + int fp1 = img[bbox[0] + off[1]]; + + if (fp0 > fp1) + { + index |= 1; + } + + off += 2; + idx++; + //if (windowIdx == 10000) + // printf("idx=%d\t fp0=%d,fp1=%d, bbox[0]=%d,off[0]=%d\t,bbox +off=%d,index=%d ***\n",idx, fp0, fp1, bbox[0], off[0], bbox[0] + off[0],index); + } + + featureVector[treeIdx] = index; + + + conf += posteriors[treeIdx * numIndices + featureVector[treeIdx]]; + //if (windowIdx==753666) + //{ + // //printf("idx=%d\t,conf=%f,posterirors[]=%f\n", windowIdx, conf, posteriors[treeIdx * numIndices + featureVector[treeIdx]]); Sleep(1000); + // printf("windowIdx=%d,featureVector[treeIdx=%d]=%d,conf=%f cpu\n", windowIdx, treeIdx, featureVector[treeIdx],conf); + // printf("treeIdx * numIndices + index=%d,posteriors[ xx] = %f.............\n", treeIdx * numIndices + index, posteriors[treeIdx * numIndices + index]); + // Sleep(1000); + //} + + + } + + detectionResult->posteriors[windowIdx] = conf; + //if (conf >= 0.5) + // { + // printf("idx=%d\t,conf=%f\n", windowIdx, conf); Sleep(1000); + // } + } + + + + bool EnsembleClassifier::clfilter(const Mat &matImg) + { + /******* ******/ + /*Begain Integral */ + /****** *******/ + int i, j; + cv::Mat t_sum; //cv::Mat sum; + cv::Mat t_sqsum;//cv::Mat sqsum; + + int vlen = 4; cl_event events[10]; + int offset = 0; + int pre_invalid = 0; + int vcols = (pre_invalid + matImg.cols + vlen - 1) / vlen; + + int w = matImg.cols, h = matImg.rows; + int depth = matImg.depth() == CV_8U ? CV_32S : CV_64F; + int type = CV_MAKE_TYPE(depth, 1); + + + t_sum.create(matImg.cols, matImg.rows, type); + iSumMat_ensemble.create(h, w, type); + + t_sqsum.create(matImg.cols, matImg.rows, CV_32FC1); + fSqreSumMat_ensemble.create(h, w, CV_32FC1); + + int sum_offset = 0; // sum.offset / vlen; + int sqsum_offset = 0; // sqsum.offset / vlen; + + + cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY, (matImg.cols) * (matImg.rows) * sizeof(uchar), NULL, NULL); + cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (matImg.cols) * (matImg.rows) * sizeof(int), NULL, NULL); + cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (matImg.cols) * (matImg.rows) * sizeof(float), NULL, NULL); + cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (matImg.cols) * (matImg.rows) * sizeof(int), NULL, NULL); + cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (matImg.cols) * (matImg.rows) * sizeof(float), NULL, NULL); + + int step; + int src_step, sum_step; + int t_sum_step, sqsum_step; + + + int rows = matImg.rows; + int cols = matImg.cols; + src_step = matImg.step; + t_sum_step = t_sum.step; + sum_step = iSumMat_ensemble.step; + sqsum_step = fSqreSumMat_ensemble.step; + + //printf(""); + status = clEnqueueWriteBuffer(commandQueue, srcdata, CL_FALSE, 0, (matImg.cols) * (matImg.rows) * sizeof(uchar), matImg.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clSetKernelArg(kernel_intgegral_cols_en, 0, sizeof(cl_mem), (void *)&srcdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&matImg.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 1, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 2, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 3, sizeof(cl_int), (void *)&offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&offset)); + status = clSetKernelArg(kernel_intgegral_cols_en, 4, sizeof(cl_int), (void *)&pre_invalid);//args.push_back(make_pair(sizeof(cl_int), (void *)&pre_invalid)); + status = clSetKernelArg(kernel_intgegral_cols_en, 5, sizeof(cl_int), (void *)&rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&matImg.rows)); + status = clSetKernelArg(kernel_intgegral_cols_en, 6, sizeof(cl_int), (void *)&cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&matImg.cols)); + status = clSetKernelArg(kernel_intgegral_cols_en, 7, sizeof(cl_int), &src_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&matImg.step)); + status = clSetKernelArg(kernel_intgegral_cols_en, 8, sizeof(cl_int), &t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + + + size_t gt[3] = { ((vcols + 1) / 2) * 256, 1, 1 }, lt[3] = { 256, 1, 1 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols_en, 1, NULL, gt, lt, 0, NULL, &events[0]); + status = clWaitForEvents(1, &events[0]); + assert(status == CL_SUCCESS); + + + status = clSetKernelArg(kernel_intgegral_rows_en, 0, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 1, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 2, sizeof(cl_mem), (void *)&sumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 3, sizeof(cl_mem), (void *)&sqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&fSqreSumMat_ensemble.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 4, sizeof(cl_int), (void *)&t_sum.rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.rows)); + status = clSetKernelArg(kernel_intgegral_rows_en, 5, sizeof(cl_int), (void *)&t_sum.cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.cols)); + status = clSetKernelArg(kernel_intgegral_rows_en, 6, sizeof(cl_int), (void *)&t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 7, sizeof(cl_int), (void *)&sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 8, sizeof(cl_int), (void *)&sqsum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&fSqreSumMat_ensemble.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 9, sizeof(cl_int), (void *)&sum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum_offset)); + status = clSetKernelArg(kernel_intgegral_rows_en, 10, sizeof(cl_int), (void *)&sqsum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum_offset)); + + size_t gt2[3] = { t_sum.cols * 32, 1, 1 }, lt2[3] = { 256, 1, 1 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_rows_en, 1, NULL, gt2, lt2, 0, NULL, &events[1]);//openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_rows", gt2, lt2, args, -1, depth); + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_rows_en_en EnqueueNDRangeKernel!" << endl; + //} + status = clWaitForEvents(1, &events[1]); + assert(status == CL_SUCCESS); + t_sum.release(); + t_sqsum.release(); + clReleaseMemObject(tsumdata); + clReleaseMemObject(tsqsumdata); +/* + * End Integral + */ -bool EnsembleClassifier::filter(int i) -{ - if(!enabled) return true; +/* + * Begain variance && ensemble filter or Posterior + */ - classifyWindow(i); + if (!enabled) return true; + int tld_window_offset_size = TLD_WINDOW_OFFSET_SIZE; + + oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), NULL, NULL); + oclbufffeatureOffsets = clCreateBuffer(context, CL_MEM_READ_ONLY, (numScales * numTrees * numFeatures * 2) * sizeof(int), NULL, NULL); + oclbuffDetectionResultfeatureVectors = clCreateBuffer(context, CL_MEM_READ_WRITE, (numWindows * numTrees)* sizeof(int), NULL, NULL); + oclbuffDetectionResultPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE, (numWindows)* sizeof(float), NULL, NULL); + oclbuffPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE, (numTrees * numIndices)* sizeof(float), NULL, NULL); + oclbuffDetectionResultVarious = clCreateBuffer(context, CL_MEM_READ_WRITE, (numWindows)* sizeof(float), NULL, NULL); + + + status = clEnqueueWriteBuffer(commandQueue, oclbuffWindowsOffset, CL_FALSE, 0, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), (void *)windowOffsets, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbufffeatureOffsets, CL_FALSE, 0, (numScales * numTrees * numFeatures * 2) * sizeof(int), (void *)featureOffsets, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbuffDetectionResultfeatureVectors, CL_FALSE, 0, (numWindows * numTrees)* sizeof(int), (void *)(void *)detectionResult->featureVectors, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbuffDetectionResultPosteriors, CL_FALSE, 0, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbuffPosteriors, CL_FALSE, 0, (numTrees * numIndices)* sizeof(float), (void *)posteriors, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbuffDetectionResultVarious, CL_FALSE, 0, (numWindows)* sizeof(float), (void *)detectionResult->variances, 0, NULL, NULL); + + + status = clSetKernelArg(variance_ensemble_kernel, 0, sizeof(int), (void *)&numIndices); + status = clSetKernelArg(variance_ensemble_kernel, 1, sizeof(int), (void *)&numTrees); + status = clSetKernelArg(variance_ensemble_kernel, 2, sizeof(int), (void *)&numFeatures); + status = clSetKernelArg(variance_ensemble_kernel, 3, sizeof(int), (void *)&tld_window_offset_size); + status = clSetKernelArg(variance_ensemble_kernel, 4, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + + status = clSetKernelArg(variance_ensemble_kernel, 5, sizeof(cl_mem), (void *)&srcdata); + status = clSetKernelArg(variance_ensemble_kernel, 6, sizeof(cl_mem), (void *)&oclbufffeatureOffsets); + status = clSetKernelArg(variance_ensemble_kernel, 7, sizeof(cl_mem), (void *)&oclbuffDetectionResultfeatureVectors); + status = clSetKernelArg(variance_ensemble_kernel, 8, sizeof(cl_mem), (void *)&oclbuffDetectionResultPosteriors); + status = clSetKernelArg(variance_ensemble_kernel, 9, sizeof(cl_mem), (void *)&oclbuffPosteriors); + + status = clSetKernelArg(variance_ensemble_kernel, 10, sizeof(cl_mem), (void *)&sumdata); + status = clSetKernelArg(variance_ensemble_kernel, 11, sizeof(cl_mem), (void *)&sqsumdata); + status = clSetKernelArg(variance_ensemble_kernel, 12, sizeof(cl_mem), (void *)&oclbuffDetectionResultVarious); + status = clSetKernelArg(variance_ensemble_kernel, 13, sizeof(float), (void *)&varianceFilter->minVar); + status = clSetKernelArg(variance_ensemble_kernel, 14, sizeof(float), (void *)&numWindows); + + + size_t global_work_size[1] = { numWindows }; + // size_t global_work_size[1] = { (numWindows-255)/256*256 }; + //size_t global_work_size[1] = { (numWindows + 16*256 -1 )/4096 * 256 }; 16windows per thread. + size_t local_work_size[1] = { 256 }; + + status = clEnqueueNDRangeKernel(commandQueue, variance_ensemble_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &events[2]); + //status = clWaitForEvents(1, &events[2]); + if (status != CL_SUCCESS) + { + + cout << "Error: EnsembleClassifier EnqueueNDRangeKernel!" << endl; + //return false; + } + + status = clEnqueueReadBuffer(commandQueue, oclbuffDetectionResultfeatureVectors, CL_FALSE, 0, (numWindows * numTrees)* sizeof(int), (void *)(void *)detectionResult->featureVectors, 0, NULL, &events[3]); + //status = clWaitForEvents(1, &events[3]); + + status = clEnqueueReadBuffer(commandQueue, oclbuffDetectionResultPosteriors, CL_FALSE, 0, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, 0, NULL, &events[4]); + //status = clWaitForEvents(1, &events[4]); + status = clEnqueueReadBuffer(commandQueue, oclbuffPosteriors, CL_FALSE, 0, (numTrees * numIndices)* sizeof(float), (void *)posteriors, 0, NULL, &events[5]); + //status = clWaitForEvents(1, &events[5]); + status = clEnqueueReadBuffer(commandQueue, oclbuffDetectionResultVarious, CL_FALSE, 0, (numWindows)* sizeof(float), (void *)detectionResult->variances, 0, NULL, &events[6]); + //status = clWaitForEvents(1, &events[6]); + + for (int i = 0; i < 6; i++) + status = clReleaseEvent(events[i]); //cout << "o" << endl; + +/* +* End ensemble filter or Posterior +*/ - if(detectionResult->posteriors[i] < 0.5) return false; + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbufffeatureOffsets); + clReleaseMemObject(oclbuffDetectionResultfeatureVectors); + clReleaseMemObject(oclbuffDetectionResultPosteriors); + clReleaseMemObject(oclbuffPosteriors); + clReleaseMemObject(oclbuffDetectionResultVarious); + clReleaseMemObject(srcdata); + clReleaseMemObject(sumdata); + clReleaseMemObject(sqsumdata); - return true; -} + return true; + } -void EnsembleClassifier::updatePosterior(int treeIdx, int idx, int positive, int amount) -{ - int arrayIndex = treeIdx * numIndices + idx; - (positive) ? positives[arrayIndex] += amount : negatives[arrayIndex] += amount; - posteriors[arrayIndex] = ((float) positives[arrayIndex]) / (positives[arrayIndex] + negatives[arrayIndex]) / (float) numTrees; -} + void EnsembleClassifier::updatePosterior(int treeIdx, int idx, int positive, int amount) + { + int arrayIndex = treeIdx * numIndices + idx; + (positive) ? positives[arrayIndex] += amount : negatives[arrayIndex] += amount; + posteriors[arrayIndex] = ((float)positives[arrayIndex]) / (positives[arrayIndex] + negatives[arrayIndex]) / (float)numTrees; + } -void EnsembleClassifier::updatePosteriors(int *featureVector, int positive, int amount) -{ + void EnsembleClassifier::updatePosteriors(int *featureVector, int positive, int amount) + { - for(int i = 0; i < numTrees; i++) - { + for (int i = 0; i < numTrees; i++) + { - int idx = featureVector[i]; - updatePosterior(i, idx, positive, amount); + int idx = featureVector[i]; + updatePosterior(i, idx, positive, amount); - } -} + } + } -void EnsembleClassifier::learn(int *boundary, int positive, int *featureVector) -{ - if(!enabled) return; + void EnsembleClassifier::learn(int *boundary, int positive, int *featureVector) + { + if (!enabled) return; - float conf = calcConfidence(featureVector); + float conf = calcConfidence(featureVector); - //Update if positive patch and confidence < 0.5 or negative and conf > 0.5 - if((positive && conf < 0.5) || (!positive && conf > 0.5)) - { - updatePosteriors(featureVector, positive, 1); - } + //Update if positive patch and confidence < 0.5 or negative and conf > 0.5 + if ((positive && conf < 0.5) || (!positive && conf > 0.5)) + { + updatePosteriors(featureVector, positive, 1); + } -} + } } /* namespace tld */ diff --git a/src/libopentld/tld/EnsembleClassifier.h b/src/libopentld/tld/EnsembleClassifier.h index 7ca0ffb9..a19eecf6 100644 --- a/src/libopentld/tld/EnsembleClassifier.h +++ b/src/libopentld/tld/EnsembleClassifier.h @@ -17,64 +17,111 @@ * */ /* - * EnsembleClassifier.h - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* EnsembleClassifier.h +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #ifndef ENSEMBLECLASSIFIER_H_ #define ENSEMBLECLASSIFIER_H_ #include +#include namespace tld { -class EnsembleClassifier -{ - const unsigned char *img; - - float calcConfidence(int *featureVector); - int calcFernFeature(int windowIdx, int treeIdx); - void calcFeatureVector(int windowIdx, int *featureVector); - void updatePosteriors(int *featureVector, int positive, int amount); -public: - bool enabled; - - //Configurable members - int numTrees; - int numFeatures; - - int imgWidthStep; - int numScales; - cv::Size *scales; - - int *windowOffsets; - int *featureOffsets; - float *features; - - int numIndices; - - float *posteriors; - int *positives; - int *negatives; - - DetectionResult *detectionResult; - - EnsembleClassifier(); - virtual ~EnsembleClassifier(); - void init(); - void initFeatureLocations(); - void initFeatureOffsets(); - void initPosteriors(); - void release(); - void nextIteration(const cv::Mat &img); - void classifyWindow(int windowIdx); - void updatePosterior(int treeIdx, int idx, int positive, int amount); - void learn(int *boundary, int positive, int *featureVector); - bool filter(int i); -}; + class EnsembleClassifier + { + const unsigned char *img; + + float calcConfidence(int *featureVector); + int calcFernFeature(int windowIdx, int treeIdx); + void calcFeatureVector(int windowIdx, int *featureVector); + void updatePosteriors(int *featureVector, int positive, int amount); + + + + public: + bool enabled; + + //Configurable members + int numTrees; + int numFeatures; + int numWindows; + + int imgWidthStep; + int numScales; + cv::Size *scales; + + int *windowOffsets; + int *featureOffsets; + float *features; + + int numIndices; + + float *posteriors; + int *positives; + int *negatives; + + DetectionResult *detectionResult; + VarianceFilter *varianceFilter; + + cv::Mat iSumMat_ensemble; + cv::Mat fSqreSumMat_ensemble; + + EnsembleClassifier(); + virtual ~EnsembleClassifier(); + void init(); + void initFeatureLocations(); + void initFeatureOffsets(); + void initPosteriors(); + void release(); + void integralImag_extract(const Mat &src, Mat &iSumMat, Mat &fSqreSumMat); + + void nextIteration(const cv::Mat &img); + void classifyWindow(int windowIdx); + void updatePosterior(int treeIdx, int idx, int positive, int amount); + void learn(int *boundary, int positive, int *featureVector); + bool filter(int i); + bool EnsembleClassifier::clfilter(const Mat &img); + void classifyWindow2(int windowIdx); + + + int convertToString(const char *filename, std::string& s); + + + //for opencl begin + + cl_platform_id platform; //the chosen platform + cl_int status; + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel variance_ensemble_kernel; + cl_kernel kernel_intgegral_cols_en, kernel_intgegral_rows_en; + + cl_mem oclbuffWindowsOffset; + cl_mem oclbufffeatureOffsets; + cl_mem oclbuffDetectionResultfeatureVectors; + cl_mem oclbuffDetectionResultPosteriors; + cl_mem oclbuffDetectionwindowFlags; + cl_mem oclbuffPosteriors; + cl_mem oclbuffDetectionResultVarious; + cl_mem oclbuffImgData; + cl_mem oclbuffII; + cl_mem oclbuffIISqure; + size_t local_work_size[1]; + + + + //for opencl end + + }; } /* namespace tld */ #endif /* ENSEMBLECLASSIFIER_H_ */ diff --git a/src/libopentld/tld/IntegralImage.h b/src/libopentld/tld/IntegralImage.h index 6e812148..9de3966a 100644 --- a/src/libopentld/tld/IntegralImage.h +++ b/src/libopentld/tld/IntegralImage.h @@ -18,64 +18,133 @@ */ /* - * IntegralImage.h - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* IntegralImage.h +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #ifndef INTEGRALIMAGE_H_ #define INTEGRALIMAGE_H_ - +#include "switch.h" #include +#include "opencv2/core/core.hpp" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/ocl/ocl.hpp" +#include "opencv2/nonfree/ocl.hpp" +#include "opencv2/calib3d/calib3d.hpp" +#include "opencv2/nonfree/nonfree.hpp" + +using namespace cv; +using namespace cv::ocl; namespace tld { -template -class IntegralImage -{ -public: - T *data; /* Array containg the entries for the integral image in row-first manner. Of size width*height. Allocated by tldAllocIntImg */ - /* width, height: Dimensions of integral image.*/ - int width; - int height; - - IntegralImage(cv::Size size) - { - data = new T[size.width * size.height]; - } - - virtual ~IntegralImage() - { - delete[] data; - } - - void calcIntImg(const cv::Mat &img, bool squared = false) - { - const unsigned char *input = (const unsigned char *)(img.data); - T *output = data; - - for(int i = 0; i < img.cols; i++) - { - for(int j = 0; j < img.rows; j++) - { - T A = (i > 0) ? output[img.cols * j + i - 1] : 0; - T B = (j > 0) ? output[img.cols * (j - 1) + i] : 0; - T C = (j > 0 && i > 0) ? output[img.cols * (j - 1) + i - 1] : 0; - T value = input[img.step * j + i]; - - if(squared) - { - value = value * value; - } - - output[img.cols * j + i] = A + B - C + value; - } - } - - } -}; + template + class IntegralImage + { + public: + T *data; /* Array containg the entries for the integral image in row-first manner. Of size width*height. Allocated by tldAllocIntImg */ + /* width, height: Dimensions of integral image.*/ + int width; + int height; + + IntegralImage(cv::Size size) + { + data = new T[(size.width) * (size.height)]; + } + + virtual ~IntegralImage() + { + delete[] data; + } + + void oclcalcIntImg(const cv::Mat &img, bool squared = false) + { + // const unsigned char *input = (const unsigned char *)(img.data); + // T *output = data; + int *input; + int *output = data; + DevicesInfo devices; + getOpenCLDevices(devices); + setDevice(devices[SELECTED_DEVICE_ID]); + oclMat img1, img2, MatSqure; + cv::Mat tempMat; + img1 = img; + + + cv::ocl::integral(img1, img2); + tempMat = img2; + + input = (int*)tempMat.data; + //double tic = cvGetTickCount(); + + for (int i = 0; i 0) ? output[img.cols * j + i - 1] : 0; + T B = (j > 0) ? output[img.cols * (j - 1) + i] : 0; + T C = (j > 0 && i > 0) ? output[img.cols * (j - 1) + i - 1] : 0; + T value = input[img.step * j + i]; + + if (squared) + { + value = value * value; + } + + output[img.cols * j + i] = A + B - C + value; + //printf("index, i=%d,j=%d,output value is %d\n", i,j,output[img.cols*j + i]); + } + } + + } + + + }; } /* namespace tld */ diff --git a/src/libopentld/tld/NNClassifier.cpp b/src/libopentld/tld/NNClassifier.cpp index 6eae808a..7340a199 100644 --- a/src/libopentld/tld/NNClassifier.cpp +++ b/src/libopentld/tld/NNClassifier.cpp @@ -17,11 +17,11 @@ * */ /* - * NNClassifier.cpp - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* NNClassifier.cpp +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #include "NNClassifier.h" @@ -34,151 +34,437 @@ using namespace cv; namespace tld { -NNClassifier::NNClassifier() -{ - thetaFP = .5; - thetaTP = .65; - - truePositives = new vector(); - falsePositives = new vector(); - -} - -NNClassifier::~NNClassifier() -{ - release(); - - delete truePositives; - delete falsePositives; -} - -void NNClassifier::release() -{ - falsePositives->clear(); - truePositives->clear(); -} - -float NNClassifier::ncc(float *f1, float *f2) -{ - double corr = 0; - double norm1 = 0; - double norm2 = 0; - - int size = TLD_PATCH_SIZE * TLD_PATCH_SIZE; - - for(int i = 0; i < size; i++) - { - corr += f1[i] * f2[i]; - norm1 += f1[i] * f1[i]; - norm2 += f2[i] * f2[i]; - } - - // normalization to <0,1> - return (corr / sqrt(norm1 * norm2) + 1) / 2.0; -} -float NNClassifier::classifyPatch(NormalizedPatch *patch) -{ - if(truePositives->empty()) - { - return 0; - } + NNClassifier::NNClassifier() + { + thetaFP = .5; + thetaTP = .65; + + truePositives = new vector(); + falsePositives = new vector(); + candidatesToNNClassifyVector = new vector(); // another method is to array + candidatesToNNClassifyIndexVector = new vector(); + pcandidatesToNNClassifyIndexArray = NULL; + pcandidatesToNNClassifyPatches = NULL; + pNNResultsArray = NULL; + kernel_nnClassifier = NULL; + pSrcTruePostiveData = NULL; + pSrcFalsePostiveData = NULL; + //nnClassifyStructInstance = new nnClassifyStruct; + } + + NNClassifier::~NNClassifier() + { + release(); + + delete truePositives; + delete falsePositives; + delete pNNResultsArray; + delete pcandidatesToNNClassifyIndexArray; + } + + void NNClassifier::release() + { + falsePositives->clear(); + truePositives->clear(); + candidatesToNNClassifyVector->clear(); + candidatesToNNClassifyIndexVector->clear(); + } + + void NNClassifier::VectorToArray() + { + + int truePostiveSize = truePositives->size(); + int falsePositiveSize = falsePositives->size(); + + pSrcTruePostiveData = new float[truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + float *p = pSrcTruePostiveData; + float *q; + for (int i = 0; i < truePostiveSize; i++) + { + + q = truePositives->at(i).values; + for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) + *p++ = *q++; + + } + + pSrcFalsePostiveData = new float[falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + p = pSrcTruePostiveData; + for (int i = 0; i < truePostiveSize; i++) + { + + q = truePositives->at(i).values; + for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) + *p++ = *q++; + } + + + + } + float NNClassifier::ncc(float *f1, float *f2) + { + double corr = 0; + double norm1 = 0; + double norm2 = 0; + + int size = TLD_PATCH_SIZE * TLD_PATCH_SIZE; + + for (int i = 0; i < size; i++) + { + corr += f1[i] * f2[i]; + norm1 += f1[i] * f1[i]; + norm2 += f2[i] * f2[i]; + } + + // normalization to <0,1> + + return (corr / sqrt(norm1 * norm2) + 1) / 2.0; + } + + float NNClassifier::classifyPatch(NormalizedPatch *patch) + { + + if (truePositives->empty()) + { + return 0; + } + + if (falsePositives->empty()) + { + return 1; + } + //printf("..................\n"); + //printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); + //printf("..................\n"); + float ccorr_max_p = 0; + + //Compare patch to positive patches + for (size_t i = 0; i < truePositives->size(); i++) + { + float ccorr = ncc(truePositives->at(i).values, patch->values); + + if (ccorr > ccorr_max_p) + { + ccorr_max_p = ccorr; + } + } + + float ccorr_max_n = 0; + + //Compare patch to negative patches + for (size_t i = 0; i < falsePositives->size(); i++) + { + float ccorr = ncc(falsePositives->at(i).values, patch->values); + + if (ccorr > ccorr_max_n) + { + ccorr_max_n = ccorr; + } + } + + float dN = 1 - ccorr_max_n; + float dP = 1 - ccorr_max_p; + + float distance = dN / (dN + dP); + return distance; + } + + float NNClassifier::classifyBB(const Mat &img, Rect *bb) + { + NormalizedPatch patch; + + tldExtractNormalizedPatchRect(img, bb, patch.values); + return classifyPatch(&patch); + + } + + float NNClassifier::classifyWindow(const Mat &img, int windowIdx) + { + NormalizedPatch patch; + + int *bbox = &windows[TLD_WINDOW_SIZE * windowIdx]; + tldExtractNormalizedPatchBB(img, bbox, patch.values); + + return classifyPatch(&patch); + } + + bool NNClassifier::filter(const Mat &img, int windowIdx) + { + if (!enabled) return true; + + float conf = classifyWindow(img, windowIdx); + //printf(" windowIdx=%d conf = %f....\n", windowIdx, conf); + + + if (conf < thetaTP) + { + return false; + } + + return true; + } + + + bool NNClassifier::clNNFilter(const cv::Mat &img) + { + + cl_event events[2]; + + int truePostiveSize = truePositives->size(); + int falsePositiveSize = falsePositives->size(); + int CandidatesToNNClassifySize = candidatesToNNClassifyIndexVector->size(); + + pcandidatesToNNClassifyPatches = new float [TLD_PATCH_SIZE * TLD_PATCH_SIZE *candidatesToNNClassifyIndexVector->size()]; + float * pPatches = pcandidatesToNNClassifyPatches; + float maxPositiveValue ; + float maxFalseValue; + pNNResultsArray = new float[ (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()]; + for (int i = 0; i < (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size(); i++) + pNNResultsArray[i] = 0.0f; + + pcandidatesToNNClassifyIndexArray = new float[candidatesToNNClassifyIndexVector->size()]; + float *pIndexArray = pcandidatesToNNClassifyIndexArray; + NormalizedPatch patch; + for (int i = 0; i < candidatesToNNClassifyIndexVector->size(); i++) + { + *pIndexArray++ = candidatesToNNClassifyIndexVector->at(i); + int *bbox = &windows[TLD_WINDOW_SIZE * i]; + tldExtractNormalizedPatchBB(img, bbox, patch.values); + float *pdest = patch.values; + for (int j = 0; j < TLD_PATCH_SIZE*TLD_PATCH_SIZE; j++) + { + *pPatches++ = *pdest++; + } + } + //printf("..................\n"); + //printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); + //printf("..................\n"); + + // case one + if (truePositives->empty()) + { + //--return 0; + //++set all ccorr_max_p equale to 1.0; + for (int i = 0, count = 0; i < this->candidatesToNNClassifyVector->size(); i++) + { + candidatesToNNClassifyVector->at(i).conf = 0.0f; + //candidatesToNNClassifyVector->at(i).index = i; + //candidatesToNNClassifyVector->at(i).flag = false; + + } + goto EndofFuction; + } + // case two + if (!truePositives->empty() && falsePositives->empty()) + { + //++ set all ccorr_max_p equale to 1.0; + for (int i = 0, count = 0; i < this->candidatesToNNClassifyVector->size(); i++) + { + candidatesToNNClassifyVector->at(i).conf = 1.0f; + //candidatesToNNClassifyVector->at(i).index = candidatesToNNClassifyIndexVector->at(i); + //candidatesToNNClassifyVector->at(i).flag = true; + + } + + goto EndofFuction; + } + //case three TODO： + //if (truePositives->empty() && !falsePositives->empty()) + + //case four + if (!truePositives->empty() && !falsePositives->empty()) + { + + int tld_window_size = TLD_WINDOW_SIZE; + //VectorToArray(); + // Begin Vector to Array + pSrcTruePostiveData = new float[truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + float *p = pSrcTruePostiveData; + float *q; + for (int i = 0; i < truePostiveSize; i++) + { + q = truePositives->at(i).values; + for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) + *p++ = *q++; + } + pSrcFalsePostiveData = new float[falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + p = pSrcFalsePostiveData; + for (int i = 0; i < falsePositiveSize; i++) + { + q = falsePositives->at(i).values; + for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) + *p++ = *q++; + } + + // End Vector to Array + oclbufferpNNResultsArray = clCreateBuffer(context, CL_MEM_READ_WRITE |CL_MEM_USE_HOST_PTR, (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()* sizeof(float), (void*)pNNResultsArray, NULL); + oclbufferSrcTruePostiveData = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcTruePostiveData, NULL); + oclbufferSrcFalsePostiveData = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcFalsePostiveData, NULL); + oclbufferCandidatesToNNClassifyPatches = clCreateBuffer(context, CL_MEM_READ_WRITE| CL_MEM_USE_HOST_PTR, TLD_PATCH_SIZE * TLD_PATCH_SIZE *candidatesToNNClassifyIndexVector->size() *sizeof(int), (void*)pcandidatesToNNClassifyPatches, NULL); + + /*Step 8: Create kernel object */ + /*Step 9: Sets Kernel arguments.*/ + status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbufferpNNResultsArray); + status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(int), (void *)&truePostiveSize); + status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(int), (void *)&falsePositiveSize); + status = clSetKernelArg(kernel_nnClassifier, 3, sizeof(int), (void *)&CandidatesToNNClassifySize); + status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(cl_mem), (void *)&oclbufferSrcTruePostiveData); + status = clSetKernelArg(kernel_nnClassifier, 5, sizeof(cl_mem), (void *)&oclbufferSrcFalsePostiveData); + status = clSetKernelArg(kernel_nnClassifier, 6, sizeof(cl_mem), (void *)&oclbufferCandidatesToNNClassifyPatches); + + //status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&TLD_WINDOW_OFFSET_SIZE); + + /*Step 10: Running the kernel.*/ + //printf("begore opencl kernel numWindows=%d\n", numWindows); + size_t global_work_size[1] = { truePostiveSize+falsePositiveSize }; + size_t local_work_size[1] = { 256 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_nnClassifier, 1, NULL, global_work_size, NULL, 0, NULL, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error:VarianceFilter EnqueueNDRangeKernel!" << endl; + //return false; + } + status = clWaitForEvents(1, &events[0]); + + if (status != CL_SUCCESS) + { + printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + + } + //cout << "o" << endl; + + + status = clEnqueueReadBuffer(commandQueue, oclbufferpNNResultsArray, CL_FALSE, 0, (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()* sizeof(float), (void *)pNNResultsArray, 0, NULL, &events[0]); + status = clWaitForEvents(1, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error:NNclassify EnqueueNDRangeKernel!" << endl; + //return false; + } + + /* printf("Reading data from GPU *************************\n"); + for (int i = 0; i < (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size(); i++) + { + printf("candidatesToNNClassifyIndexVector->size()=%d， detectionResult[%d] is %f\n", candidatesToNNClassifyIndexVector->size(),i, pNNResultsArray[i]); + + + } + */ + for (int i = 0; i < candidatesToNNClassifyIndexVector->size();i++) + { + maxPositiveValue = pNNResultsArray[i]; + maxFalseValue = pNNResultsArray[i+ truePostiveSize*candidatesToNNClassifyIndexVector->size()]; + for (int j = 1; j < truePostiveSize; j++) + { + if ( maxPositiveValue < pNNResultsArray[j*candidatesToNNClassifyIndexVector->size()]) + maxPositiveValue = pNNResultsArray[j*candidatesToNNClassifyIndexVector->size()]; + + } + for (int j = truePostiveSize; j < truePostiveSize+falsePositiveSize; j++) + { + if (maxFalseValue < pNNResultsArray[j*candidatesToNNClassifyIndexVector->size()]) + maxFalseValue = pNNResultsArray[j*candidatesToNNClassifyIndexVector->size()]; + + } + float dN = 1 - maxFalseValue; + float dP = 1 - maxPositiveValue; + + float conf = dN / (dN + dP); + // if (conf > thetaTP) + //{ + // return false; + //} + candidatesToNNClassifyVector->at(i).conf = conf; + candidatesToNNClassifyVector->at(i).index = candidatesToNNClassifyIndexVector->at(i); + candidatesToNNClassifyVector->at(i).flag = true; + + } + + + + //status = clReleaseEvent(events[0]); + //status = clReleaseEvent(events[1]); + + //printf("end using GPU*************************\n"); + //for (int i = 0; i < numWindows; i++) + // if(detectionResult->windowFlags[i]==1) + // printf("detectionResult[%d] is %d\n", i,detectionResult->windowFlags[i]); + + /*Step 11: Read the cout put back to host memory.*/ + //status = clEnqueueReadBuffer(commandQueue, outputBuffer, CL_TRUE, 0, 12 * sizeof(char), output, 0, NULL, NULL); - if(falsePositives->empty()) - { - return 1; - } - float ccorr_max_p = 0; + delete pSrcTruePostiveData; + pSrcTruePostiveData = NULL; + delete pSrcFalsePostiveData; + pSrcFalsePostiveData = NULL; + delete pNNResultsArray; + pNNResultsArray = NULL; + delete pcandidatesToNNClassifyIndexArray; + pcandidatesToNNClassifyIndexArray = NULL; + delete pcandidatesToNNClassifyPatches; + pcandidatesToNNClassifyPatches = NULL; - //Compare patch to positive patches - for(size_t i = 0; i < truePositives->size(); i++) - { - float ccorr = ncc(truePositives->at(i).values, patch->values); - if(ccorr > ccorr_max_p) - { - ccorr_max_p = ccorr; - } - } + // clReleaseKernel(kernel_nnClassifier); + // clReleaseProgram(program); + // clReleaseMemObject(oclbufferSrcData); + // clReleaseMemObject(oclbufferWindows); + // clReleaseMemObject(oclbuffercandidatesToNNClassifyIndexArray); + clReleaseMemObject(oclbufferpNNResultsArray); + clReleaseMemObject(oclbufferCandidatesToNNClassifyPatches); + - float ccorr_max_n = 0; - //Compare patch to negative patches - for(size_t i = 0; i < falsePositives->size(); i++) - { - float ccorr = ncc(falsePositives->at(i).values, patch->values); + return true; - if(ccorr > ccorr_max_n) - { - ccorr_max_n = ccorr; - } - } - float dN = 1 - ccorr_max_n; - float dP = 1 - ccorr_max_p; + /*end copy */ - float distance = dN / (dN + dP); - return distance; -} -float NNClassifier::classifyBB(const Mat &img, Rect *bb) -{ - NormalizedPatch patch; + + } - tldExtractNormalizedPatchRect(img, bb, patch.values); - return classifyPatch(&patch); -} -float NNClassifier::classifyWindow(const Mat &img, int windowIdx) -{ - NormalizedPatch patch; - int *bbox = &windows[TLD_WINDOW_SIZE * windowIdx]; - tldExtractNormalizedPatchBB(img, bbox, patch.values); - return classifyPatch(&patch); -} -bool NNClassifier::filter(const Mat &img, int windowIdx) -{ - if(!enabled) return true; - float conf = classifyWindow(img, windowIdx); - if(conf < thetaTP) - { - return false; - } - return true; -} + float ccorr_max_p = 0; + EndofFuction: + return true; + + + } -void NNClassifier::learn(vector patches) -{ - //TODO: Randomization might be a good idea here - for(size_t i = 0; i < patches.size(); i++) - { + void NNClassifier::learn(vector patches) + { + //TODO: Randomization might be a good idea here + for (size_t i = 0; i < patches.size(); i++) + { - NormalizedPatch patch = patches[i]; + NormalizedPatch patch = patches[i]; - float conf = classifyPatch(&patch); + float conf = classifyPatch(&patch); - if(patch.positive && conf <= thetaTP) - { - truePositives->push_back(patch); - } + if (patch.positive && conf <= thetaTP) + { + truePositives->push_back(patch); + } - if(!patch.positive && conf >= thetaFP) - { - falsePositives->push_back(patch); - } - } + if (!patch.positive && conf >= thetaFP) + { + falsePositives->push_back(patch); + } + } -} + } } /* namespace tld */ diff --git a/src/libopentld/tld/NNClassifier.h b/src/libopentld/tld/NNClassifier.h index 662c6edf..e3da4078 100644 --- a/src/libopentld/tld/NNClassifier.h +++ b/src/libopentld/tld/NNClassifier.h @@ -29,6 +29,7 @@ #include #include +#include #include "NormalizedPatch.h" #include "DetectionResult.h" @@ -36,19 +37,41 @@ namespace tld { + +class nnClassifyStruct +{ +public: +float conf; +int index; +bool flag; +}; + + class NNClassifier { float ncc(float *f1, float *f2); + public: bool enabled; int *windows; + //Configurable members + + int numWindows; float thetaFP; float thetaTP; DetectionResult *detectionResult; std::vector* falsePositives; - std::vector* truePositives; + std::vector* truePositives; + float *pNNResultsArray; + float *pcandidatesToNNClassifyIndexArray; + float *pcandidatesToNNClassifyPatches; + std::vector *candidatesToNNClassifyIndexVector; + std::vector * candidatesToNNClassifyVector; + + nnClassifyStruct nnClassifyStructInstance; + NNClassifier(); virtual ~NNClassifier(); @@ -58,6 +81,46 @@ class NNClassifier float classifyWindow(const cv::Mat &img, int windowIdx); void learn(std::vector patches); bool filter(const cv::Mat &img, int windowIdx); + + + + //for opencl begin in NNclassifier + bool clNNFilter(const cv::Mat &img); + // variable members + cl_platform_id platform; //the chosen platform + cl_int status; + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + + + void VectorToArray(); + /* + * class NormalizedPatch + * { + * public: + * float values[TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + * bool positive; + *}; + */ + float *pSrcTruePostiveData; //float values[TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + float *pSrcFalsePostiveData; + + cl_mem oclbufferSrcData; + cl_mem oclbufferWindows; + cl_mem oclbuffercandidatesToNNClassifyIndexArray; + cl_mem oclbufferpNNResultsArray; + cl_mem oclbufferSrcTruePostiveData; // device memory to SrcTruePostiveData + cl_mem oclbufferSrcFalsePostiveData; // device memory to SrcFalsePostiveData + cl_mem oclbufferCandidatesToNNClassifyPatches; // candidatest to NNClassify patches from host to devices. + + + cl_kernel kernel_nnClassifier; + + + //for opencl end in NNclassifier + }; } /* namespace tld */ diff --git a/src/libopentld/tld/TLD.cpp b/src/libopentld/tld/TLD.cpp index e366a93c..8b9d788f 100644 --- a/src/libopentld/tld/TLD.cpp +++ b/src/libopentld/tld/TLD.cpp @@ -17,648 +17,970 @@ * */ /* - * TLD.cpp - * - * Created on: Nov 17, 2011 - * Author: Georg Nebehay - */ +* TLD.cpp +* +* Created on: Nov 17, 2011 +* Author: Georg Nebehay +*/ #include "TLD.h" #include - +#include +#include #include "NNClassifier.h" #include "TLDUtil.h" -using namespace std; -using namespace cv; - -namespace tld -{ - -TLD::TLD() -{ - trackerEnabled = true; - detectorEnabled = true; - learningEnabled = true; - alternating = false; - valid = false; - wasValid = false; - learning = false; - currBB = NULL; - prevBB = new Rect(0,0,0,0); - - detectorCascade = new DetectorCascade(); - nnClassifier = detectorCascade->nnClassifier; - - medianFlowTracker = new MedianFlowTracker(); -} - -TLD::~TLD() -{ - storeCurrentData(); - - if(currBB) - { - delete currBB; - currBB = NULL; - } - - if(detectorCascade) - { - delete detectorCascade; - detectorCascade = NULL; - } - - if(medianFlowTracker) - { - delete medianFlowTracker; - medianFlowTracker = NULL; - } - - if(prevBB) - { - delete prevBB; - prevBB = NULL; - } -} - -void TLD::release() -{ - detectorCascade->release(); - medianFlowTracker->cleanPreviousData(); - - if(currBB) - { - delete currBB; - currBB = NULL; - } -} - -void TLD::storeCurrentData() -{ - prevImg.release(); - prevImg = currImg; //Store old image (if any) - if(currBB)//Store old bounding box (if any) - { - prevBB->x = currBB->x; - prevBB->y = currBB->y; - prevBB->width = currBB->width; - prevBB->height = currBB->height; - } - else - { - prevBB->x = 0; - prevBB->y = 0; - prevBB->width = 0; - prevBB->height = 0; - } - - detectorCascade->cleanPreviousData(); //Reset detector results - medianFlowTracker->cleanPreviousData(); - - wasValid = valid; -} - -void TLD::selectObject(const Mat &img, Rect *bb) -{ - //Delete old object - detectorCascade->release(); - - detectorCascade->objWidth = bb->width; - detectorCascade->objHeight = bb->height; - - //Init detector cascade - detectorCascade->init(); - - currImg = img; - if(currBB) - { - delete currBB; - currBB = NULL; - } - currBB = tldCopyRect(bb); - currConf = 1; - valid = true; - - initialLearning(); - -} - -void TLD::processImage(const Mat &img) -{ - storeCurrentData(); - Mat grey_frame; - cvtColor(img, grey_frame, CV_BGR2GRAY); - currImg = grey_frame; // Store new image , right after storeCurrentData(); - - if(trackerEnabled) - { - medianFlowTracker->track(prevImg, currImg, prevBB); - } - - if(detectorEnabled && (!alternating || medianFlowTracker->trackerBB == NULL)) - { - detectorCascade->detect(grey_frame); - } - - fuseHypotheses(); - - learn(); - -} - -void TLD::fuseHypotheses() -{ - Rect *trackerBB = medianFlowTracker->trackerBB; - int numClusters = detectorCascade->detectionResult->numClusters; - Rect *detectorBB = detectorCascade->detectionResult->detectorBB; - - if(currBB) - { - delete currBB; - currBB = NULL; - } - currConf = 0; - valid = false; - - float confDetector = 0; - - if(numClusters == 1) - { - confDetector = nnClassifier->classifyBB(currImg, detectorBB); - } - - if(trackerBB != NULL) - { - float confTracker = nnClassifier->classifyBB(currImg, trackerBB); - if(currBB) - { - delete currBB; - currBB = NULL; - } - - if(numClusters == 1 && confDetector > confTracker && tldOverlapRectRect(*trackerBB, *detectorBB) < 0.5) - { - - currBB = tldCopyRect(detectorBB); - currConf = confDetector; - } - else - { - currBB = tldCopyRect(trackerBB); - currConf = confTracker; - - if(confTracker > nnClassifier->thetaTP) - { - valid = true; - } - else if(wasValid && confTracker > nnClassifier->thetaFP) - { - valid = true; - } - } - } - else if(numClusters == 1) - { - if(currBB) - { - delete currBB; - currBB = NULL; - } - currBB = tldCopyRect(detectorBB); - currConf = confDetector; - } - - /* - float var = CalculateVariance(patch.values, nn->patch_size*nn->patch_size); - - if(var < min_var) { //TODO: Think about incorporating this - printf("%f, %f: Variance too low \n", var, classifier->min_var); - valid = 0; - }*/ -} - -void TLD::initialLearning() -{ - learning = true; //This is just for display purposes - - DetectionResult *detectionResult = detectorCascade->detectionResult; - - detectorCascade->detect(currImg); - - //This is the positive patch - NormalizedPatch patch; - tldExtractNormalizedPatchRect(currImg, currBB, patch.values); - patch.positive = 1; - - float initVar = tldCalcVariance(patch.values, TLD_PATCH_SIZE * TLD_PATCH_SIZE); - detectorCascade->varianceFilter->minVar = initVar / 2; - - - float *overlap = new float[detectorCascade->numWindows]; - tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); - //Add all bounding boxes with high overlap - vector< pair > positiveIndices; - vector negativeIndices; +#include +#include "switch.h" +#include - //First: Find overlapping positive and negative patches - for(int i = 0; i < detectorCascade->numWindows; i++) - { - - if(overlap[i] > 0.6) - { - positiveIndices.push_back(pair(i, overlap[i])); - } - - if(overlap[i] < 0.2) - { - float variance = detectionResult->variances[i]; - - if(!detectorCascade->varianceFilter->enabled || variance > detectorCascade->varianceFilter->minVar) //TODO: This check is unnecessary if minVar would be set before calling detect. - { - negativeIndices.push_back(i); - } - } - } - - sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); - - vector patches; - - patches.push_back(patch); //Add first patch to patch list - - int numIterations = std::min(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) - - for(int i = 0; i < numIterations; i++) - { - int idx = positiveIndices.at(i).first; - //Learn this bounding box - //TODO: Somewhere here image warping might be possible - detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); - } - - srand(1); //TODO: This is not guaranteed to affect random_shuffle - - random_shuffle(negativeIndices.begin(), negativeIndices.end()); - - //Choose 100 random patches for negative examples - for(size_t i = 0; i < std::min(100, negativeIndices.size()); i++) - { - int idx = negativeIndices.at(i); - - NormalizedPatch patch; - tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); - patch.positive = 0; - patches.push_back(patch); - } - - detectorCascade->nnClassifier->learn(patches); - - delete[] overlap; - -} +using namespace std; +using namespace cv; -//Do this when current trajectory is valid -void TLD::learn() +namespace tld { - if(!learningEnabled || !valid || !detectorEnabled) - { - learning = false; - return; - } - learning = true; + TLD::TLD() + { + trackerEnabled = true; + detectorEnabled = true; + learningEnabled = true; + alternating = false; + valid = false; + wasValid = false; + learning = false; + currBB = NULL; + prevBB = new Rect(0, 0, 0, 0); + + detectorCascade = new DetectorCascade(); + nnClassifier = detectorCascade->nnClassifier; + + //TLD tld opencl enviroments + numWindows = detectorCascade->numWindows; + platform = detectorCascade->platform; + devices = detectorCascade->devices; + context = detectorCascade->context; + commandQueue = detectorCascade->commandQueue; + + + + medianFlowTracker = new MedianFlowTracker(); + + int tld_window_size = TLD_WINDOW_SIZE; + char *kernelName = "..\\..\\..\\src\\libopentld\\kernel\\cltldOverlapRect_kernel.cpp"; + + string sourceStr; + status = convertToString2(kernelName, sourceStr); + + const char *source = sourceStr.c_str(); + size_t sourceSize[] = { strlen(source) }; + program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); + + /*Step 6: Build program. */ + status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + if (status != CL_SUCCESS) + { + size_t log_size; + char* program_log; + clGetProgramBuildInfo(program, devices[SELECTED_DEVICE_ID], CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size); + program_log = (char*)malloc(log_size + 1); + clGetProgramBuildInfo(program, devices[SELECTED_DEVICE_ID], CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL); + printf("%s\n", program_log); + free(program_log); + + + cout << "DetectorCascade::oclBuildKernelError:clBuildProgram !" << endl; + //return false; + } + } + + TLD::~TLD() + { + storeCurrentData(); + + if (currBB) + { + delete currBB; + currBB = NULL; + } + + if (detectorCascade) + { + delete detectorCascade; + detectorCascade = NULL; + } + + if (medianFlowTracker) + { + delete medianFlowTracker; + medianFlowTracker = NULL; + } + + if (prevBB) + { + delete prevBB; + prevBB = NULL; + } + } + + void TLD::release() + { + detectorCascade->release(); + medianFlowTracker->cleanPreviousData(); + + if (currBB) + { + delete currBB; + currBB = NULL; + } + } + int TLD::convertToString2(const char *filename, std::string& s) + { + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); + + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } + + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return false; + } + + void TLD::storeCurrentData() + { + prevImg.release(); + prevImg = currImg; //Store old image (if any) + if (currBB)//Store old bounding box (if any) + { + prevBB->x = currBB->x; + prevBB->y = currBB->y; + prevBB->width = currBB->width; + prevBB->height = currBB->height; + } + else + { + prevBB->x = 0; + prevBB->y = 0; + prevBB->width = 0; + prevBB->height = 0; + } + + detectorCascade->cleanPreviousData(); //Reset detector results + medianFlowTracker->cleanPreviousData(); + + wasValid = valid; + } + + void TLD::selectObject(const Mat &img, Rect *bb) + { + +#if opencl + printf("*******************I am using OpenCL***********************\n"); +#else + printf("*******************I am using CPU***********************\n"); +#endif + //Delete old object + detectorCascade->release(); + + detectorCascade->objWidth = bb->width; + detectorCascade->objHeight = bb->height; + + //Init detector cascade + detectorCascade->init(); + + currImg = img; + if (currBB) + { + delete currBB; + currBB = NULL; + } + currBB = tldCopyRect(bb); + currConf = 1; + valid = true; + + initialLearning(); + + } + + void TLD::processImage(const Mat &img) + { + storeCurrentData(); + Mat grey_frame; + + double tic, toc; + cvtColor(img, grey_frame, CV_BGR2GRAY); + currImg = grey_frame; // Store new image , right after storeCurrentData(); + + if (trackerEnabled) + { + medianFlowTracker->track(prevImg, currImg, prevBB); + } + + if (detectorEnabled && (!alternating || medianFlowTracker->trackerBB == NULL)) + { +#if PrintTime_detect + tic = cvGetTickCount(); +#endif +#if clDetect + detectorCascade->cldetect(grey_frame); +#else + detectorCascade->detect(grey_frame); +#endif + +#if PrintTime_detect + toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the tldDect time used is %f************\n", toc);//float fps = 1 / toc; +#endif + + } + +#if PrintTime_fuseHypotheses + tic = cvGetTickCount(); +#endif + + + fuseHypotheses(); +#if PrintTime_fuseHypotheses + toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the PrintTime_fuseHypotheses time used is %f************\n", toc);//float fps = 1 / toc; +#endif +#if PrintTime_learn + tic = cvGetTickCount(); + +#endif + learn(); +#if PrintTime_learn + toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the PrintTime_learn time used is %f************\n", toc);//float fps = 1 / toc; +#endif + + } + + void TLD::fuseHypotheses() + { + Rect *trackerBB = medianFlowTracker->trackerBB; + int numClusters = detectorCascade->detectionResult->numClusters; + Rect *detectorBB = detectorCascade->detectionResult->detectorBB; + + if (currBB) + { + delete currBB; + currBB = NULL; + } + currConf = 0; + valid = false; + + float confDetector = 0; + + if (numClusters == 1) + { + confDetector = nnClassifier->classifyBB(currImg, detectorBB); + } + + if (trackerBB != NULL) + { + float confTracker = nnClassifier->classifyBB(currImg, trackerBB); + if (currBB) + { + delete currBB; + currBB = NULL; + } + + if (numClusters == 1 && confDetector > confTracker && tldOverlapRectRect(*trackerBB, *detectorBB) < 0.5) + { + + currBB = tldCopyRect(detectorBB); + currConf = confDetector; + //printf("FrameNum=%d,currConf = confDetector=%f....\n", detectorCascade->FramNum, currConf); + } + else + { + currBB = tldCopyRect(trackerBB); + currConf = confTracker; + //printf("FrameNum=%d,currConf = confTracker = %f....\n", detectorCascade->FramNum, currConf); + if (confTracker > nnClassifier->thetaTP) + { + valid = true; + } + else if (wasValid && confTracker > nnClassifier->thetaFP) + { + valid = true; + } + } + } + else if (numClusters == 1) + { + if (currBB) + { + delete currBB; + currBB = NULL; + } + currBB = tldCopyRect(detectorBB); + currConf = confDetector; + //printf("FrameNum=%d,numClusters == 1 currConf = confTracker=%f....\n", detectorCascade->FramNum,currConf); + } + + /* + float var = CalculateVariance(patch.values, nn->patch_size*nn->patch_size); + + if(var < min_var) { //TODO: Think about incorporating this + printf("%f, %f: Variance too low \n", var, classifier->min_var); + valid = 0; + }*/ + } + void TLD::cltldOverlapRect_self(int *windows, int numWindows, Rect *boundary, float *overlap) + { + + + cl_event events[1]; + int tld_window_size = TLD_WINDOW_SIZE; + + int bb0, bb1, bb2, bb3; + + bb0 = boundary->x; + bb1 = boundary->y; + bb2 = boundary->width; + bb3 = boundary->height; + //float *overlap = new float[detectorCascade->numWindows]; + + + cl_mem oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_SIZE * numWindows) * sizeof(int), (void *)windows, NULL); + cl_mem oclbuffOverlap = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)overlap, NULL); + kernel = clCreateKernel(program, "cltldOverlapRect", NULL); + + + status = clSetKernelArg(kernel, 0, sizeof(int), (void *)&bb0); + status = clSetKernelArg(kernel, 1, sizeof(int), (void *)&bb1); + status = clSetKernelArg(kernel, 2, sizeof(int), (void *)&bb2); + status = clSetKernelArg(kernel, 3, sizeof(int), (void *)&bb3); + + status = clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&oclbuffOverlap); + status = clSetKernelArg(kernel, 6, sizeof(int), (void *)&tld_window_size); + + + /*Step 10: Running the kernel.*/ + //printf("begore opencl kernel numWindows=%d\n", numWindows); + size_t global_work_size[1] = { numWindows }; + size_t local_work_size[1] = { 256 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error:cltldOverlapRect_self EnqueueNDRangeKernel!" << endl; + //return false; + } + status = clWaitForEvents(1, &events[0]); + + if (status != CL_SUCCESS) + { + printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + + } + //cout << "o" << endl; + status = clReleaseEvent(events[0]); + + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbuffOverlap); + + //return true; + + } + void TLD::cltldOverlapRect(int *windows, int numWindows, Rect *boundary, float *overlap) + { + + //printf("begin using initial clFilter*************************\n"); + cl_event events[1]; + int tld_window_size = TLD_WINDOW_SIZE; + char *kernelName = "..\\..\\..\\src\\libopentld\\kernel\\cltldOverlapRect_kernel.cpp"; + string sourceStr; + status = convertToString2(kernelName, sourceStr); + const char *source = sourceStr.c_str(); + size_t sourceSize[] = { strlen(source) }; + program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); + + /*Step 6: Build program. */ + status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + //printf("mid using EnsembleClassifier clFilter*************************\n"); + if (status != CL_SUCCESS) + { + cout << "overlap Error: Getting platforms!" << endl; + //return false; + } + int bb0, bb1, bb2, bb3; + + bb0 = boundary->x; + bb1 = boundary->y; + bb2 = boundary->width; + bb3 = boundary->height; + //float *overlap = new float[detectorCascade->numWindows]; + + + //int *windows, int numWindows, Rect *boundary, float *overlap + cl_mem oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_SIZE * numWindows) * sizeof(int), (void *)windows, NULL); + cl_mem oclbuffOverlap = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)overlap, NULL); + + + /*Step 8: Create kernel object */ + kernel = clCreateKernel(program, "cltldOverlapRect", NULL); + /*Step 9: Sets Kernel arguments.*/ + status = clSetKernelArg(kernel, 0, sizeof(int), (void *)&bb0); + status = clSetKernelArg(kernel, 1, sizeof(int), (void *)&bb1); + status = clSetKernelArg(kernel, 2, sizeof(int), (void *)&bb2); + status = clSetKernelArg(kernel, 3, sizeof(int), (void *)&bb3); + + status = clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&oclbuffOverlap); + status = clSetKernelArg(kernel, 6, sizeof(int), (void *)&tld_window_size); + + + /*Step 10: Running the kernel.*/ + //printf("begore opencl kernel numWindows=%d\n", numWindows); + size_t global_work_size[1] = { numWindows }; + size_t local_work_size[1] = { 256 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error: EnqueueNDRangeKernel!" << endl; + //return false; + } + status = clWaitForEvents(1, &events[0]); + + if (status != CL_SUCCESS) + { + printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + + } + //cout << "o" << endl; + status = clReleaseEvent(events[0]); + + clReleaseKernel(kernel); + clReleaseProgram(program); + + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbuffOverlap); + + //return true; + + } + + void TLD::initialLearning() + { + learning = true; //This is just for display purposes + + DetectionResult *detectionResult = detectorCascade->detectionResult; +#if clDetect + detectorCascade->cldetect(currImg); +#else + detectorCascade->detect(currImg); +#endif + // + //This is the positive patch + NormalizedPatch patch; + tldExtractNormalizedPatchRect(currImg, currBB, patch.values); + patch.positive = 1; + + float initVar = tldCalcVariance(patch.values, TLD_PATCH_SIZE * TLD_PATCH_SIZE); + detectorCascade->varianceFilter->minVar = initVar / 2; + + + float *overlap = new float[detectorCascade->numWindows]; + // double tic = cvGetTickCount(); +#if clOverlap + cltldOverlapRect_self(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); +#else + tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); +#endif + // double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + // toc = toc / 1000000; + // printf("*****the tldOverlapRect time used is %f************\n", toc);//float fps = 1 / toc; + + // printf("****************i am 248426 overlap[248426]= %f*****************\n", overlap[248426]); + // printf("****************i am 249404 overlap[248426]= %f*****************\n", overlap[249404]); + //Add all bounding boxes with high overlap + + vector< pair > positiveIndices; + vector negativeIndices; + + //First: Find overlapping positive and negative patches + + for (int i = 0; i < detectorCascade->numWindows; i++) + { - DetectionResult *detectionResult = detectorCascade->detectionResult; + if (overlap[i] > 0.6) + { + positiveIndices.push_back(pair(i, overlap[i])); - if(!detectionResult->containsValidData) - { - detectorCascade->detect(currImg); - } + } - //This is the positive patch - NormalizedPatch patch; - tldExtractNormalizedPatchRect(currImg, currBB, patch.values); + if (overlap[i] < 0.2) + { + float variance = detectionResult->variances[i]; + //printf("variance=%f\n", variance); + + if (!detectorCascade->varianceFilter->enabled || variance > detectorCascade->varianceFilter->minVar) //TODO: This check is unnecessary if minVar would be set before calling detect. + { + //printf("variance=%f\n", variance); + negativeIndices.push_back(i); + + } + } + } + //cout << "xxxxnegativeIndices.size()xxx" << negativeIndices.size() << endl; + // FILE *fp = fopen("C:\\Users\\kevin\\Desktop\\compResult\\opentld_opencl_overlap.txt", "a+"); + // for (int i = 0; i < negativeIndices.size();i++) + //fprintf(fp, "windowIdx=%d,overlap=%f\n", i, overlap[i]); + // fprintf(fp, "%d\t%f\n", i, negativeIndices.at(i)); + // fclose(fp); + + sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); + + vector patches; + + patches.push_back(patch); //Add first patch to patch list + + int numIterations = std::min(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) + + for (int i = 0; i < numIterations; i++) + { + int idx = positiveIndices.at(i).first; + //Learn this bounding box + //TODO: Somewhere here image warping might be possible + detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); + } + + srand(1); //TODO: This is not guaranteed to affect random_shuffle + + random_shuffle(negativeIndices.begin(), negativeIndices.end()); + + //Choose 100 random patches for negative examples + for (size_t i = 0; i < std::min(100, negativeIndices.size()); i++) + { + int idx = negativeIndices.at(i); + + NormalizedPatch patch; + tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); + patch.positive = 0; + patches.push_back(patch); + } + + detectorCascade->nnClassifier->learn(patches); + + delete[] overlap; + + } - float *overlap = new float[detectorCascade->numWindows]; - tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); + //Do this when current trajectory is valid + void TLD::learn() + { + if (!learningEnabled || !valid || !detectorEnabled) + { + learning = false; + return; + } - //Add all bounding boxes with high overlap + learning = true; - vector > positiveIndices; - vector negativeIndices; - vector negativeIndicesForNN; + DetectionResult *detectionResult = detectorCascade->detectionResult; + + if (!detectionResult->containsValidData) + { + +#if PrintTime_detect + double tic = cvGetTickCount(); +#endif +#if clDetect + detectorCascade->cldetect(currImg); +#else + detectorCascade->detect(currImg); +#endif - //First: Find overlapping positive and negative patches +#if PrintTime_detect + double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the tldDect time used is %f************\n", toc);//float fps = 1 / toc; +#endif + } + + //This is the positive patch + NormalizedPatch patch; + tldExtractNormalizedPatchRect(currImg, currBB, patch.values); + + float *overlap = new float[detectorCascade->numWindows]; + +#if PrintTime_overlap + double tic = cvGetTickCount(); +#endif +#if clOverlap + cltldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); +#else + tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); +#endif +#if PrintTime_overlap + double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the tldOverlapRect time used is %f************\n", toc);//float fps = 1 / toc; +#endif + + //Add all bounding boxes with high overlap + + vector > positiveIndices; + vector negativeIndices; + vector negativeIndicesForNN; + + //First: Find overlapping positive and negative patches + + for (int i = 0; i < detectorCascade->numWindows; i++) + { + + if (overlap[i] > 0.6) + { + positiveIndices.push_back(pair(i, overlap[i])); + //FILE *fp = fopen("opentld_sort_overlap.txt", "a+"); + //fprintf(fp, "windowIdx=%d,overlap=%f\n", i, overlap[i]); + //fclose(fp); + } + + if (overlap[i] < 0.2) + { + if (!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) //Should be 0.5 according to the paper + { + negativeIndices.push_back(i); + negativeIndicesForNN.push_back(i); + } + + //if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) + //{ + // negativeIndicesForNN.push_back(i); + //} + + } + } + + sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); + + vector patches; + + patch.positive = 1; + patches.push_back(patch); + //TODO: Flip + + + int numIterations = std::min(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) + + for (size_t i = 0; i < negativeIndices.size(); i++) + { + int idx = negativeIndices.at(i); + //TODO: Somewhere here image warping might be possible + detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], false, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); + } + + //TODO: Randomization might be a good idea + for (int i = 0; i < numIterations; i++) + { + int idx = positiveIndices.at(i).first; + //TODO: Somewhere here image warping might be possible + detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); + } + + for (size_t i = 0; i < negativeIndicesForNN.size(); i++) + { + int idx = negativeIndicesForNN.at(i); + + NormalizedPatch patch; + tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); + patch.positive = 0; + patches.push_back(patch); + } + + detectorCascade->nnClassifier->learn(patches); + //cout << "xxxxnegativeIndices.size()xxx ....." << negativeIndices.size() << "negativeIndicesForNN.size()=...;;" << negativeIndicesForNN.size() << endl; + //cout << "NN has now " << detectorCascade->nnClassifier->truePositives->size() << " positives and " << detectorCascade->nnClassifier->falsePositives->size() << " negatives.\n"; + + delete[] overlap; + /*debug float *p; + p = detectorCascade->ensembleClassifier->posteriors; + for (int i = 0; i < 10; i++) + { + for (int j = 0; j < 8192; j++) + { + + if (p[i * 8192 + j] > 0.11) + printf("the tree is %d, the ensembleClassifier posteriors is %f\n",i , p[i * 8192 + j]); + } + printf("xxxxxxxxxxxxxxxxxxxxxx the tree is %d\n", i); + } + */ + } + + typedef struct + { + int index; + int P; + int N; + } TldExportEntry; + + void TLD::writeToFile(const char *path) + { + NNClassifier *nn = detectorCascade->nnClassifier; + EnsembleClassifier *ec = detectorCascade->ensembleClassifier; + + FILE *file = fopen(path, "w"); + fprintf(file, "#Tld ModelExport\n"); + fprintf(file, "%d #width\n", detectorCascade->objWidth); + fprintf(file, "%d #height\n", detectorCascade->objHeight); + fprintf(file, "%f #min_var\n", detectorCascade->varianceFilter->minVar); + fprintf(file, "%d #Positive Sample Size\n", nn->truePositives->size()); + + + + for (size_t s = 0; s < nn->truePositives->size(); s++) + { + float *imageData = nn->truePositives->at(s).values; + + for (int i = 0; i < TLD_PATCH_SIZE; i++) + { + for (int j = 0; j < TLD_PATCH_SIZE; j++) + { + fprintf(file, "%f ", imageData[i * TLD_PATCH_SIZE + j]); + } + + fprintf(file, "\n"); + } + } + + fprintf(file, "%d #Negative Sample Size\n", nn->falsePositives->size()); + + for (size_t s = 0; s < nn->falsePositives->size(); s++) + { + float *imageData = nn->falsePositives->at(s).values; + + for (int i = 0; i < TLD_PATCH_SIZE; i++) + { + for (int j = 0; j < TLD_PATCH_SIZE; j++) + { + fprintf(file, "%f ", imageData[i * TLD_PATCH_SIZE + j]); + } + + fprintf(file, "\n"); + } + } + + fprintf(file, "%d #numtrees\n", ec->numTrees); + detectorCascade->numTrees = ec->numTrees; + fprintf(file, "%d #numFeatures\n", ec->numFeatures); + detectorCascade->numFeatures = ec->numFeatures; + + for (int i = 0; i < ec->numTrees; i++) + { + fprintf(file, "#Tree %d\n", i); + + for (int j = 0; j < ec->numFeatures; j++) + { + float *features = ec->features + 4 * ec->numFeatures * i + 4 * j; + fprintf(file, "%f %f %f %f # Feature %d\n", features[0], features[1], features[2], features[3], j); + } + + //Collect indices + vector list; + + for (int index = 0; index < pow(2.0f, ec->numFeatures); index++) + { + int p = ec->positives[i * ec->numIndices + index]; + + if (p != 0) + { + TldExportEntry entry; + entry.index = index; + entry.P = p; + entry.N = ec->negatives[i * ec->numIndices + index]; + list.push_back(entry); + } + } + + fprintf(file, "%d #numLeaves\n", list.size()); - for(int i = 0; i < detectorCascade->numWindows; i++) - { + for (size_t j = 0; j < list.size(); j++) + { + TldExportEntry entry = list.at(j); + fprintf(file, "%d %d %d\n", entry.index, entry.P, entry.N); + } + } - if(overlap[i] > 0.6) - { - positiveIndices.push_back(pair(i, overlap[i])); - } + fclose(file); - if(overlap[i] < 0.2) - { - if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) //Should be 0.5 according to the paper - { - negativeIndices.push_back(i); - } + } - if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) - { - negativeIndicesForNN.push_back(i); - } - - } - } - - sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); - - vector patches; - - patch.positive = 1; - patches.push_back(patch); - //TODO: Flip - - - int numIterations = std::min(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) - - for(size_t i = 0; i < negativeIndices.size(); i++) - { - int idx = negativeIndices.at(i); - //TODO: Somewhere here image warping might be possible - detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], false, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); - } - - //TODO: Randomization might be a good idea - for(int i = 0; i < numIterations; i++) - { - int idx = positiveIndices.at(i).first; - //TODO: Somewhere here image warping might be possible - detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); - } - - for(size_t i = 0; i < negativeIndicesForNN.size(); i++) - { - int idx = negativeIndicesForNN.at(i); - - NormalizedPatch patch; - tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); - patch.positive = 0; - patches.push_back(patch); - } - - detectorCascade->nnClassifier->learn(patches); - - //cout << "NN has now " << detectorCascade->nnClassifier->truePositives->size() << " positives and " << detectorCascade->nnClassifier->falsePositives->size() << " negatives.\n"; - - delete[] overlap; -} - -typedef struct -{ - int index; - int P; - int N; -} TldExportEntry; - -void TLD::writeToFile(const char *path) -{ - NNClassifier *nn = detectorCascade->nnClassifier; - EnsembleClassifier *ec = detectorCascade->ensembleClassifier; - - FILE *file = fopen(path, "w"); - fprintf(file, "#Tld ModelExport\n"); - fprintf(file, "%d #width\n", detectorCascade->objWidth); - fprintf(file, "%d #height\n", detectorCascade->objHeight); - fprintf(file, "%f #min_var\n", detectorCascade->varianceFilter->minVar); - fprintf(file, "%d #Positive Sample Size\n", nn->truePositives->size()); - - - - for(size_t s = 0; s < nn->truePositives->size(); s++) - { - float *imageData = nn->truePositives->at(s).values; - - for(int i = 0; i < TLD_PATCH_SIZE; i++) - { - for(int j = 0; j < TLD_PATCH_SIZE; j++) - { - fprintf(file, "%f ", imageData[i * TLD_PATCH_SIZE + j]); - } - - fprintf(file, "\n"); - } - } - - fprintf(file, "%d #Negative Sample Size\n", nn->falsePositives->size()); - - for(size_t s = 0; s < nn->falsePositives->size(); s++) - { - float *imageData = nn->falsePositives->at(s).values; - - for(int i = 0; i < TLD_PATCH_SIZE; i++) - { - for(int j = 0; j < TLD_PATCH_SIZE; j++) - { - fprintf(file, "%f ", imageData[i * TLD_PATCH_SIZE + j]); - } - - fprintf(file, "\n"); - } - } - - fprintf(file, "%d #numtrees\n", ec->numTrees); - detectorCascade->numTrees = ec->numTrees; - fprintf(file, "%d #numFeatures\n", ec->numFeatures); - detectorCascade->numFeatures = ec->numFeatures; - - for(int i = 0; i < ec->numTrees; i++) - { - fprintf(file, "#Tree %d\n", i); - - for(int j = 0; j < ec->numFeatures; j++) - { - float *features = ec->features + 4 * ec->numFeatures * i + 4 * j; - fprintf(file, "%f %f %f %f # Feature %d\n", features[0], features[1], features[2], features[3], j); - } - - //Collect indices - vector list; - - for(int index = 0; index < pow(2.0f, ec->numFeatures); index++) - { - int p = ec->positives[i * ec->numIndices + index]; - - if(p != 0) - { - TldExportEntry entry; - entry.index = index; - entry.P = p; - entry.N = ec->negatives[i * ec->numIndices + index]; - list.push_back(entry); - } - } - - fprintf(file, "%d #numLeaves\n", list.size()); - - for(size_t j = 0; j < list.size(); j++) - { - TldExportEntry entry = list.at(j); - fprintf(file, "%d %d %d\n", entry.index, entry.P, entry.N); - } - } - - fclose(file); - -} - -void TLD::readFromFile(const char *path) -{ - release(); + void TLD::readFromFile(const char *path) + { + release(); - NNClassifier *nn = detectorCascade->nnClassifier; - EnsembleClassifier *ec = detectorCascade->ensembleClassifier; + NNClassifier *nn = detectorCascade->nnClassifier; + EnsembleClassifier *ec = detectorCascade->ensembleClassifier; - FILE *file = fopen(path, "r"); + FILE *file = fopen(path, "r"); - if(file == NULL) - { - printf("Error: Model not found: %s\n", path); - exit(1); - } + if (file == NULL) + { + printf("Error: Model not found: %s\n", path); + exit(1); + } - int MAX_LEN = 255; - char str_buf[255]; - fgets(str_buf, MAX_LEN, file); /*Skip line*/ + int MAX_LEN = 255; + char str_buf[255]; + fgets(str_buf, MAX_LEN, file); /*Skip line*/ - fscanf(file, "%d \n", &detectorCascade->objWidth); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - fscanf(file, "%d \n", &detectorCascade->objHeight); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%d \n", &detectorCascade->objWidth); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%d \n", &detectorCascade->objHeight); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - fscanf(file, "%f \n", &detectorCascade->varianceFilter->minVar); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%f \n", &detectorCascade->varianceFilter->minVar); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - int numPositivePatches; - fscanf(file, "%d \n", &numPositivePatches); - fgets(str_buf, MAX_LEN, file); /*Skip line*/ + int numPositivePatches; + fscanf(file, "%d \n", &numPositivePatches); + fgets(str_buf, MAX_LEN, file); /*Skip line*/ - for(int s = 0; s < numPositivePatches; s++) - { - NormalizedPatch patch; + for (int s = 0; s < numPositivePatches; s++) + { + NormalizedPatch patch; - for(int i = 0; i < 15; i++) //Do 15 times - { + for (int i = 0; i < 15; i++) //Do 15 times + { - fgets(str_buf, MAX_LEN, file); /*Read sample*/ + fgets(str_buf, MAX_LEN, file); /*Read sample*/ - char *pch; - pch = strtok(str_buf, " \n"); - int j = 0; + char *pch; + pch = strtok(str_buf, " \n"); + int j = 0; - while(pch != NULL) - { - float val = atof(pch); - patch.values[i * TLD_PATCH_SIZE + j] = val; + while (pch != NULL) + { + float val = atof(pch); + patch.values[i * TLD_PATCH_SIZE + j] = val; - pch = strtok(NULL, " \n"); + pch = strtok(NULL, " \n"); - j++; - } - } + j++; + } + } - nn->truePositives->push_back(patch); - } + nn->truePositives->push_back(patch); + } - int numNegativePatches; - fscanf(file, "%d \n", &numNegativePatches); - fgets(str_buf, MAX_LEN, file); /*Skip line*/ + int numNegativePatches; + fscanf(file, "%d \n", &numNegativePatches); + fgets(str_buf, MAX_LEN, file); /*Skip line*/ - for(int s = 0; s < numNegativePatches; s++) - { - NormalizedPatch patch; + for (int s = 0; s < numNegativePatches; s++) + { + NormalizedPatch patch; - for(int i = 0; i < 15; i++) //Do 15 times - { + for (int i = 0; i < 15; i++) //Do 15 times + { - fgets(str_buf, MAX_LEN, file); /*Read sample*/ + fgets(str_buf, MAX_LEN, file); /*Read sample*/ - char *pch; - pch = strtok(str_buf, " \n"); - int j = 0; + char *pch; + pch = strtok(str_buf, " \n"); + int j = 0; - while(pch != NULL) - { - float val = atof(pch); - patch.values[i * TLD_PATCH_SIZE + j] = val; + while (pch != NULL) + { + float val = atof(pch); + patch.values[i * TLD_PATCH_SIZE + j] = val; - pch = strtok(NULL, " \n"); + pch = strtok(NULL, " \n"); - j++; - } - } + j++; + } + } - nn->falsePositives->push_back(patch); - } + nn->falsePositives->push_back(patch); + } - fscanf(file, "%d \n", &ec->numTrees); - detectorCascade->numTrees = ec->numTrees; - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%d \n", &ec->numTrees); + detectorCascade->numTrees = ec->numTrees; + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - fscanf(file, "%d \n", &ec->numFeatures); - detectorCascade->numFeatures = ec->numFeatures; - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%d \n", &ec->numFeatures); + detectorCascade->numFeatures = ec->numFeatures; + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - int size = 2 * 2 * ec->numFeatures * ec->numTrees; - ec->features = new float[size]; - ec->numIndices = pow(2.0f, ec->numFeatures); - ec->initPosteriors(); + int size = 2 * 2 * ec->numFeatures * ec->numTrees; + ec->features = new float[size]; + ec->numIndices = pow(2.0f, ec->numFeatures); + ec->initPosteriors(); - for(int i = 0; i < ec->numTrees; i++) - { - fgets(str_buf, MAX_LEN, file); /*Skip line*/ + for (int i = 0; i < ec->numTrees; i++) + { + fgets(str_buf, MAX_LEN, file); /*Skip line*/ - for(int j = 0; j < ec->numFeatures; j++) - { - float *features = ec->features + 4 * ec->numFeatures * i + 4 * j; - fscanf(file, "%f %f %f %f", &features[0], &features[1], &features[2], &features[3]); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - } + for (int j = 0; j < ec->numFeatures; j++) + { + float *features = ec->features + 4 * ec->numFeatures * i + 4 * j; + fscanf(file, "%f %f %f %f", &features[0], &features[1], &features[2], &features[3]); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + } - /* read number of leaves*/ - int numLeaves; - fscanf(file, "%d \n", &numLeaves); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + /* read number of leaves*/ + int numLeaves; + fscanf(file, "%d \n", &numLeaves); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - for(int j = 0; j < numLeaves; j++) - { - TldExportEntry entry; - fscanf(file, "%d %d %d \n", &entry.index, &entry.P, &entry.N); - ec->updatePosterior(i, entry.index, 1, entry.P); - ec->updatePosterior(i, entry.index, 0, entry.N); - } - } + for (int j = 0; j < numLeaves; j++) + { + TldExportEntry entry; + fscanf(file, "%d %d %d \n", &entry.index, &entry.P, &entry.N); + ec->updatePosterior(i, entry.index, 1, entry.P); + ec->updatePosterior(i, entry.index, 0, entry.N); + } + } - detectorCascade->initWindowsAndScales(); - detectorCascade->initWindowOffsets(); + detectorCascade->initWindowsAndScales(); + detectorCascade->initWindowOffsets(); - detectorCascade->propagateMembers(); + detectorCascade->propagateMembers(); - detectorCascade->initialised = true; + detectorCascade->initialised = true; - ec->initFeatureOffsets(); + ec->initFeatureOffsets(); - fclose(file); -} + fclose(file); + } } /* namespace tld */ diff --git a/src/libopentld/tld/TLD.h b/src/libopentld/tld/TLD.h index 9723d091..e477d5f5 100644 --- a/src/libopentld/tld/TLD.h +++ b/src/libopentld/tld/TLD.h @@ -18,11 +18,11 @@ */ /* - * TLD.h - * - * Created on: Nov 17, 2011 - * Author: Georg Nebehay - */ +* TLD.h +* +* Created on: Nov 17, 2011 +* Author: Georg Nebehay +*/ #ifndef TLD_H_ #define TLD_H_ @@ -31,42 +31,61 @@ #include "MedianFlowTracker.h" #include "DetectorCascade.h" +#include namespace tld { -class TLD -{ - void storeCurrentData(); - void fuseHypotheses(); - void learn(); - void initialLearning(); -public: - bool trackerEnabled; - bool detectorEnabled; - bool learningEnabled; - bool alternating; + class TLD + { + void storeCurrentData(); + void fuseHypotheses(); + void learn(); + void initialLearning(); + public: + bool trackerEnabled; + bool detectorEnabled; + bool learningEnabled; + bool alternating; + + MedianFlowTracker *medianFlowTracker; + DetectorCascade *detectorCascade; + NNClassifier *nnClassifier; + bool valid; + bool wasValid; + cv::Mat prevImg; + cv::Mat currImg; + cv::Rect *prevBB; + cv::Rect *currBB; + float currConf; + bool learning; + + TLD(); + virtual ~TLD(); + void release(); + void selectObject(const cv::Mat &img, cv::Rect *bb); + void processImage(const cv::Mat &img); + void writeToFile(const char *path); + void readFromFile(const char *path); + - MedianFlowTracker *medianFlowTracker; - DetectorCascade *detectorCascade; - NNClassifier *nnClassifier; - bool valid; - bool wasValid; - cv::Mat prevImg; - cv::Mat currImg; - cv::Rect *prevBB; - cv::Rect *currBB; - float currConf; - bool learning; + //for opencl begin + int numWindows; + cl_platform_id platform; //the chosen platform + cl_int status; + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel kernel; + //for opencl end + void cltldOverlapRect(int *windows, int numWindows, Rect *boundary, float *overlap); + void cltldOverlapRect_self(int *windows, int numWindows, Rect *boundary, float *overlap); + int convertToString2(const char *filename, std::string& s); - TLD(); - virtual ~TLD(); - void release(); - void selectObject(const cv::Mat &img, cv::Rect *bb); - void processImage(const cv::Mat &img); - void writeToFile(const char *path); - void readFromFile(const char *path); -}; + }; } /* namespace tld */ #endif /* TLD_H_ */ diff --git a/src/libopentld/tld/TLDUtil.cpp b/src/libopentld/tld/TLDUtil.cpp index 3587a037..b430771b 100644 --- a/src/libopentld/tld/TLDUtil.cpp +++ b/src/libopentld/tld/TLDUtil.cpp @@ -16,234 +16,312 @@ * along with OpenTLD. If not, see . * */ - +#include #include "TLDUtil.h" #include "NormalizedPatch.h" #include "DetectorCascade.h" - +#include "omp.h" using namespace std; using namespace cv; namespace tld { -void tldRectToPoints(Rect rect, CvPoint *p1, CvPoint *p2) -{ - p1->x = rect.x; - p1->y = rect.y; - p2->x = rect.x + rect.width; - p2->y = rect.y + rect.height; -} - -void tldBoundingBoxToPoints(int *bb, CvPoint *p1, CvPoint *p2) -{ - p1->x = bb[0]; - p1->y = bb[1]; - p2->x = bb[0] + bb[2]; - p2->y = bb[1] + bb[3]; -} - - - -//Returns mean-normalized patch, image must be greyscale -void tldNormalizeImg(const Mat &img, float *output) -{ - int size = TLD_PATCH_SIZE; - - Mat result; - resize(img, result, cvSize(size, size)); //Default is bilinear - - float mean = 0; - - unsigned char *imgData = (unsigned char *)result.data; - - for(int i = 0; i < 15; i++) - { - for(int j = 0; j < 15; j++) - { - mean += imgData[j * result.step + i]; - } - } - - mean /= size * size; - - - for(int i = 0; i < size; i++) - { - for(int j = 0; j < size; j++) - { - output[j * 15 + i] = imgData[j * result.step + i] - mean; - } - } - -} - -CvRect tldBoundaryToRect(int *boundary) -{ - return Rect(boundary[0], boundary[1], boundary[2], boundary[3]); -} - -void tldExtractSubImage(const Mat &img, Mat &subImage, CvRect rect) -{ - subImage = img(rect).clone(); -} + void tldRectToPoints(Rect rect, CvPoint *p1, CvPoint *p2) + { + p1->x = rect.x; + p1->y = rect.y; + p2->x = rect.x + rect.width; + p2->y = rect.y + rect.height; + } + + void tldBoundingBoxToPoints(int *bb, CvPoint *p1, CvPoint *p2) + { + p1->x = bb[0]; + p1->y = bb[1]; + p2->x = bb[0] + bb[2]; + p2->y = bb[1] + bb[3]; + } -void tldExtractSubImage(const Mat &img, Mat &subImage, int *boundary) -{ - tldExtractSubImage(img, subImage, tldBoundaryToRect(boundary)); -} -void tldExtractNormalizedPatch(const Mat &img, int x, int y, int w, int h, float *output) -{ - Mat subImage; - tldExtractSubImage(img, subImage, Rect(x, y, w, h)); - tldNormalizeImg(subImage, output); -} -//TODO: Rename -void tldExtractNormalizedPatchBB(const Mat &img, int *boundary, float *output) -{ - int x, y, w, h; - tldExtractDimsFromArray(boundary, &x, &y, &w, &h); - tldExtractNormalizedPatch(img, x, y, w, h, output); -} + //Returns mean-normalized patch, image must be greyscale + void tldNormalizeImg(const Mat &img, float *output) + { + int size = TLD_PATCH_SIZE; -void tldExtractNormalizedPatchRect(const Mat &img, Rect *rect, float *output) -{ - tldExtractNormalizedPatch(img, rect->x, rect->y, rect->width, rect->height, output); -} + Mat result; + resize(img, result, cvSize(size, size)); //Default is bilinear -float CalculateMean(float *value, int n) -{ + float mean = 0; - float sum = 0; + unsigned char *imgData = (unsigned char *)result.data; - for(int i = 0; i < n; i++) - sum += value[i]; + for (int i = 0; i < 15; i++) + { + for (int j = 0; j < 15; j++) + { + mean += imgData[j * result.step + i]; + } + } - return (sum / n); + mean /= size * size; -} -float tldCalcVariance(float *value, int n) -{ + for (int i = 0; i < size; i++) + { + for (int j = 0; j < size; j++) + { + output[j * 15 + i] = imgData[j * result.step + i] - mean; + } + } - float mean = CalculateMean(value, n); - float temp = 0; + } - for(int i = 0; i < n; i++) - { - temp += (value[i] - mean) * (value[i] - mean) ; - } + CvRect tldBoundaryToRect(int *boundary) + { + return Rect(boundary[0], boundary[1], boundary[2], boundary[3]); + } - return temp / n; + void tldExtractSubImage(const Mat &img, Mat &subImage, CvRect rect) + { + subImage = img(rect).clone(); + } -} - - -float tldBBOverlap(int *bb1, int *bb2) -{ + void tldExtractSubImage(const Mat &img, Mat &subImage, int *boundary) + { + tldExtractSubImage(img, subImage, tldBoundaryToRect(boundary)); + } - if(bb1[0] > bb2[0] + bb2[2]) - { - return 0.0; - } - - if(bb1[1] > bb2[1] + bb2[3]) - { - return 0.0; - } - - if(bb1[0] + bb1[2] < bb2[0]) - { - return 0.0; - } - - if(bb1[1] + bb1[3] < bb2[1]) - { - return 0.0; - } - - int colInt = min(bb1[0] + bb1[2], bb2[0] + bb2[2]) - max(bb1[0], bb2[0]); - int rowInt = min(bb1[1] + bb1[3], bb2[1] + bb2[3]) - max(bb1[1], bb2[1]); - - int intersection = colInt * rowInt; - int area1 = bb1[2] * bb1[3]; - int area2 = bb2[2] * bb2[3]; - return intersection / (float)(area1 + area2 - intersection); -} - -void tldOverlapOne(int *windows, int numWindows, int index, vector * indices, float *overlap) -{ - - for(size_t i = 0; i < indices->size(); i++) - { - - overlap[i] = tldBBOverlap(&windows[TLD_WINDOW_SIZE * index], &windows[TLD_WINDOW_SIZE * indices->at(i)]); - } - -} - -float tldOverlapRectRect(Rect r1, Rect r2) -{ - int bb1[4]; - int bb2[4]; - tldRectToArray(r1, bb1); - tldRectToArray(r2, bb2); - return tldBBOverlap(bb1, bb2); - -} - -Rect *tldCopyRect(Rect *r) -{ - Rect *r2 = new Rect(); - r2->x = r->x; - r2->y = r->y; - r2->width = r->width; - r2->height = r->height; - return r2; -} - -void tldOverlapRect(int *windows, int numWindows, Rect *boundary, float *overlap) -{ - int bb[4]; - bb[0] = boundary->x; - bb[1] = boundary->y; - bb[2] = boundary->width; - bb[3] = boundary->height; - - tldOverlap(windows, numWindows, bb, overlap); -} - -void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap) -{ - - for(int i = 0; i < numWindows; i++) - { - - overlap[i] = tldBBOverlap(boundary, &windows[TLD_WINDOW_SIZE * i]); - } - -} - - - - -bool tldSortByOverlapDesc(pair bb1 , pair bb2) -{ - return bb1.second > bb2.second; -} - -//Checks whether bb1 is completely inside bb2 -int tldIsInside(int *bb1, int *bb2) -{ + void tldExtractNormalizedPatch(const Mat &img, int x, int y, int w, int h, float *output) + { + Mat subImage; + tldExtractSubImage(img, subImage, Rect(x, y, w, h)); + tldNormalizeImg(subImage, output); + } - if(bb1[0] > bb2[0] && bb1[1] > bb2[1] && bb1[0] + bb1[2] < bb2[0] + bb2[2] && bb1[1] + bb1[3] < bb2[1] + bb2[3]) - { - return 1; - } - else return 0; + //TODO: Rename + void tldExtractNormalizedPatchBB(const Mat &img, int *boundary, float *output) + { + int x, y, w, h; + tldExtractDimsFromArray(boundary, &x, &y, &w, &h); + tldExtractNormalizedPatch(img, x, y, w, h, output); + } + + void tldExtractNormalizedPatchRect(const Mat &img, Rect *rect, float *output) + { + tldExtractNormalizedPatch(img, rect->x, rect->y, rect->width, rect->height, output); + } + + float CalculateMean(float *value, int n) + { + + float sum = 0; + + for (int i = 0; i < n; i++) + sum += value[i]; + + return (sum / n); + + } + + float tldCalcVariance(float *value, int n) + { + + float mean = CalculateMean(value, n); + float temp = 0; + float temp2 = 0; + for (int i = 0; i < n; i++) + { + temp2 = (value[i] - mean); + temp += temp2*temp2; + } + + return temp / n; + + } + + + float tldBBOverlap(int *bb1, int *bb2) + { + + if (bb1[0] > bb2[0] + bb2[2]) + { + return 0.0; + } + + if (bb1[1] > bb2[1] + bb2[3]) + { + return 0.0; + } + + if (bb1[0] + bb1[2] < bb2[0]) + { + return 0.0; + } + + if (bb1[1] + bb1[3] < bb2[1]) + { + return 0.0; + } + + int colInt = min(bb1[0] + bb1[2], bb2[0] + bb2[2]) - max(bb1[0], bb2[0]); + int rowInt = min(bb1[1] + bb1[3], bb2[1] + bb2[3]) - max(bb1[1], bb2[1]); + + int intersection = colInt * rowInt; + int area1 = bb1[2] * bb1[3]; + int area2 = bb2[2] * bb2[3]; + return intersection / (float)(area1 + area2 - intersection); + } + + void tldOverlapOne(int *windows, int numWindows, int index, vector * indices, float *overlap) + { + + for (size_t i = 0; i < indices->size(); i++) + { + + overlap[i] = tldBBOverlap(&windows[TLD_WINDOW_SIZE * index], &windows[TLD_WINDOW_SIZE * indices->at(i)]); + } + + } + + float tldOverlapRectRect(Rect r1, Rect r2) + { + int bb1[4]; + int bb2[4]; + tldRectToArray(r1, bb1); + tldRectToArray(r2, bb2); + return tldBBOverlap(bb1, bb2); + + } + + Rect *tldCopyRect(Rect *r) + { + Rect *r2 = new Rect(); + r2->x = r->x; + r2->y = r->y; + r2->width = r->width; + r2->height = r->height; + return r2; + } + + void tldOverlapRect(int *windows, int numWindows, Rect *boundary, float *overlap) + { + int bb[4]; + bb[0] = boundary->x; + bb[1] = boundary->y; + bb[2] = boundary->width; + bb[3] = boundary->height; + + tldOverlap(windows, numWindows, bb, overlap); + } + //expanded/unfolded by xqc + void tldOverlapRect_unfold(int *windows, int numWindows, Rect *boundary, float *overlap) + { + int bb1[4], *bb2; + bb1[0] = boundary->x; + bb1[1] = boundary->y; + bb1[2] = boundary->width; + bb1[3] = boundary->height; + //tldOverlap(windows, numWindows, bb, overlap); + //void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap) + //for (int i = 0; i < numWindows; i++) + //{ + // overlap[i] = tldBBOverlap(boundary, &windows[TLD_WINDOW_SIZE * i]); + //} + + + + for (int i = 0; i < numWindows; i++) + { + bb2 = windows + TLD_WINDOW_SIZE * i; + + //overlap[i] = tldBBOverlap(bb, &windows[TLD_WINDOW_SIZE * i]); + //float tldBBOverlap(int *bb1, int *bb2) + + + if (bb1[0] > bb2[0] + bb2[2]) + { + overlap[i] = 0.0; + continue; + } + + if (bb1[1] > bb2[1] + bb2[3]) + { + overlap[i] = 0.0; + continue; + } + + if (bb1[0] + bb1[2] < bb2[0]) + { + overlap[i] = 0.0; + continue; + } + + if (bb1[1] + bb1[3] < bb2[1]) + { + overlap[i] = 0.0; + continue; + } + else + { + int colInt = min(bb1[0] + bb1[2], bb2[0] + bb2[2]) - max(bb1[0], bb2[0]); + int rowInt = min(bb1[1] + bb1[3], bb2[1] + bb2[3]) - max(bb1[1], bb2[1]); + + int intersection = colInt * rowInt; + int area1 = bb1[2] * bb1[3]; + int area2 = bb2[2] * bb2[3]; + overlap[i] = intersection / (float)(area1 + area2 - intersection); + } + //if (overlap[i] != 0.0) + //{ + // printf("the index is %d, overlap[%d]=%f\n", i, i, overlap[i]); + // Sleep(1000); + //} + + //printf("windowIdx =%d ,bb[0]=%d,bb2[1]=%d,bb[2]=%d,bb[3]=%d\n", i, bb2[0], bb2[1], bb2[2], bb2[3]); Sleep(1000); + } + + } + void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap) + { + + //omp_set_num_threads(4); +#pragma omp parallel for num_threads(8) + for (int i = 0; i < numWindows; i++) + { + + // int thread = omp_get_thread_num(); + // printf("thread=%d\n",thread) ; + //float temp; + //temp = tldBBOverlap(boundary, &windows[TLD_WINDOW_SIZE * i]); + //overlap[i] = temp; + overlap[i] = tldBBOverlap(boundary, &windows[TLD_WINDOW_SIZE * i]); + + + } + + + } + + + + + bool tldSortByOverlapDesc(pair bb1, pair bb2) + { + return bb1.second > bb2.second; + } + + //Checks whether bb1 is completely inside bb2 + int tldIsInside(int *bb1, int *bb2) + { + + if (bb1[0] > bb2[0] && bb1[1] > bb2[1] && bb1[0] + bb1[2] < bb2[0] + bb2[2] && bb1[1] + bb1[3] < bb2[1] + bb2[3]) + { + return 1; + } + else return 0; -} + } } /* End Namespace */ diff --git a/src/libopentld/tld/TLDUtil.h b/src/libopentld/tld/TLDUtil.h index dece7168..40cbdefd 100644 --- a/src/libopentld/tld/TLDUtil.h +++ b/src/libopentld/tld/TLDUtil.h @@ -18,11 +18,11 @@ */ /* - * util.h - * - * Created on: 30.05.2011 - * Author: Georg Nebehay - */ +* util.h +* +* Created on: 30.05.2011 +* Author: Georg Nebehay +*/ #ifndef TLDUTIL_H_ #define TLDUTIL_H_ @@ -34,86 +34,87 @@ namespace tld { -template -void tldConvertBB(T1 *src, T2 *dest) -{ - dest[0] = src[0]; - dest[1] = src[1]; - dest[2] = src[2]; - dest[3] = src[3]; -} - -template -void tldCopyBB(T *src, T *dest) -{ - tldConvertBB(src, dest); -} - -template -void tldCopyBoundaryToArray(T x, T y, T width, T height, T *array) -{ - array[0] = x; - array[1] = y; - array[2] = width; - array[3] = height; -} - -template -void tldExtractDimsFromArray(T *boundary, T *x, T *y, T *width, T *height) -{ - *x = boundary[0]; - *y = boundary[1]; - *width = boundary[2]; - *height = boundary[3]; -} - -template -void tldRectToArray(cv::Rect rect, T *boundary) -{ - boundary[0] = rect.x; - boundary[1] = rect.y; - boundary[2] = rect.width; - boundary[3] = rect.height; -} - -template -cv::Rect tldArrayToRect(T *boundary) -{ - cv::Rect rect; - rect.x = boundary[0]; - rect.y = boundary[1]; - rect.width = boundary[2]; - rect.height = boundary[3]; - - return rect; -} - - -int tldIsInside(int *bb1, int *bb2); -void tldRectToPoints(CvRect rect, CvPoint *p1, CvPoint *p2); -void tldBoundingBoxToPoints(int *bb, CvPoint *p1, CvPoint *p2); - -void tldNormalizeImg(const cv::Mat &img, float *result, int size); - -void tldExtractNormalizedPatch(const cv::Mat &img, int x, int y, int w, int h, float *output); -void tldExtractNormalizedPatchBB(const cv::Mat &img, int *boundary, float *output); -void tldExtractNormalizedPatchRect(const cv::Mat &img, cv::Rect *rect, float *output); -void tldExtractSubImage(const cv::Mat &img, cv::Mat &subImage, int *boundary); -void tldExtractSubImage(const cv::Mat &img, cv::Mat &subImage, int x, int y, int w, int h); - -float tldCalcMean(float *value, int n); -float tldCalcVariance(float *value, int n); - -bool tldSortByOverlapDesc(std::pair bb1 , std::pair bb2); -cv::Rect *tldCopyRect(cv::Rect *r); - -//TODO: Change function names -float tldOverlapRectRect(cv::Rect r1, cv::Rect r2); -void tldOverlapOne(int *windows, int numWindows, int index, std::vector * indices, float *overlap); -void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap); -void tldOverlapRect(int *windows, int numWindows, cv::Rect *boundary, float *overlap); - -float tldCalcVariance(float *value, int n); + template + void tldConvertBB(T1 *src, T2 *dest) + { + dest[0] = src[0]; + dest[1] = src[1]; + dest[2] = src[2]; + dest[3] = src[3]; + } + + template + void tldCopyBB(T *src, T *dest) + { + tldConvertBB(src, dest); + } + + template + void tldCopyBoundaryToArray(T x, T y, T width, T height, T *array) + { + array[0] = x; + array[1] = y; + array[2] = width; + array[3] = height; + } + + template + void tldExtractDimsFromArray(T *boundary, T *x, T *y, T *width, T *height) + { + *x = boundary[0]; + *y = boundary[1]; + *width = boundary[2]; + *height = boundary[3]; + } + + template + void tldRectToArray(cv::Rect rect, T *boundary) + { + boundary[0] = rect.x; + boundary[1] = rect.y; + boundary[2] = rect.width; + boundary[3] = rect.height; + } + + template + cv::Rect tldArrayToRect(T *boundary) + { + cv::Rect rect; + rect.x = boundary[0]; + rect.y = boundary[1]; + rect.width = boundary[2]; + rect.height = boundary[3]; + + return rect; + } + + + int tldIsInside(int *bb1, int *bb2); + void tldRectToPoints(CvRect rect, CvPoint *p1, CvPoint *p2); + void tldBoundingBoxToPoints(int *bb, CvPoint *p1, CvPoint *p2); + + void tldNormalizeImg(const cv::Mat &img, float *result, int size); + + void tldExtractNormalizedPatch(const cv::Mat &img, int x, int y, int w, int h, float *output); + void tldExtractNormalizedPatchBB(const cv::Mat &img, int *boundary, float *output); + void tldExtractNormalizedPatchRect(const cv::Mat &img, cv::Rect *rect, float *output); + void tldExtractSubImage(const cv::Mat &img, cv::Mat &subImage, int *boundary); + void tldExtractSubImage(const cv::Mat &img, cv::Mat &subImage, int x, int y, int w, int h); + + float tldCalcMean(float *value, int n); + float tldCalcVariance(float *value, int n); + + bool tldSortByOverlapDesc(std::pair bb1, std::pair bb2); + cv::Rect *tldCopyRect(cv::Rect *r); + + //TODO: Change function names + float tldOverlapRectRect(cv::Rect r1, cv::Rect r2); + void tldOverlapOne(int *windows, int numWindows, int index, std::vector * indices, float *overlap); + void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap); + void tldOverlapRect(int *windows, int numWindows, cv::Rect *boundary, float *overlap); + void tldOverlapRect_unfold(int *windows, int numWindows, cv::Rect *boundary, float *overlap); + + float tldCalcVariance(float *value, int n); #endif /* UTIL_H_ */ diff --git a/src/libopentld/tld/VarianceFilter.cpp b/src/libopentld/tld/VarianceFilter.cpp index 12858bc7..0bbcbef4 100644 --- a/src/libopentld/tld/VarianceFilter.cpp +++ b/src/libopentld/tld/VarianceFilter.cpp @@ -18,84 +18,517 @@ */ /* - * VarianceFilter.cpp - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* VarianceFilter.cpp +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #include "VarianceFilter.h" #include "IntegralImage.h" #include "DetectorCascade.h" +#include "TLDUtil.h" +#include +#include +#include "switch.h" + +using namespace std; + + using namespace cv; +using namespace std; namespace tld { -VarianceFilter::VarianceFilter() -{ - enabled = true; - minVar = 0; - integralImg = NULL; - integralImg_squared = NULL; -} + VarianceFilter::VarianceFilter() + { + enabled = true; + minVar = 0; + integralImg = NULL; + integralImg_squared = NULL; -VarianceFilter::~VarianceFilter() -{ - release(); -} + } -void VarianceFilter::release() -{ - if(integralImg != NULL) delete integralImg; + VarianceFilter::~VarianceFilter() + { + release(); + } - integralImg = NULL; + void VarianceFilter::release() + { + if (integralImg != NULL) delete integralImg; - if(integralImg_squared != NULL) delete integralImg_squared; + integralImg = NULL; - integralImg_squared = NULL; -} + if (integralImg_squared != NULL) delete integralImg_squared; -float VarianceFilter::calcVariance(int *off) -{ + integralImg_squared = NULL; - int *ii1 = integralImg->data; - long long *ii2 = integralImg_squared->data; + } - float mX = (ii1[off[3]] - ii1[off[2]] - ii1[off[1]] + ii1[off[0]]) / (float) off[5]; //Sum of Area divided by area - float mX2 = (ii2[off[3]] - ii2[off[2]] - ii2[off[1]] + ii2[off[0]]) / (float) off[5]; - return mX2 - mX * mX; -} + int VarianceFilter::convertToString(const char *filename, std::string& s) + { + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); -void VarianceFilter::nextIteration(const Mat &img) -{ - if(!enabled) return; + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } - release(); + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return false; + } - integralImg = new IntegralImage(img.size()); - integralImg->calcIntImg(img); + float VarianceFilter::calcVariance(int *off) + { - integralImg_squared = new IntegralImage(img.size()); - integralImg_squared->calcIntImg(img, true); -} + int *ii1 = integralImg->data; + float *ii2 = integralImg_squared->data; + + float mX = (ii1[off[3]] - ii1[off[2]] - ii1[off[1]] + ii1[off[0]]) / (float)off[5]; //Sum of Area divided by area + float mX2 = (ii2[off[3]] - ii2[off[2]] - ii2[off[1]] + ii2[off[0]]) / (float)off[5]; + return mX2 - mX * mX; + } + + void VarianceFilter::integralImag_opencv(const Mat &img) + { + + + + ///DevicesInfo devices; + //getOpenCLDevices(devices); + //setDevice(devices[SELECTED_DEVICE_ID]); + oclMat img1, Sum, sqsum; + + img1 = img; + + + cv::ocl::integral(img1, Sum, sqsum); + iSumMat = Sum; + fSqreSumMat = sqsum; + + //int *p2, *q2; + //float *fp2, *fq2; + //p2 = (int*) iSumMat.data; + //fp2 = (float*)fSqreSumMat.data; + + //for (int i = 0; i < 1; i++) + //{ + // for (int m = 0; m < 10; m++) + // { + // q2 = p2 + i*img.cols + m; + // fq2 = fp2 + i*img.cols + m; + // // printf("%f ", (float)(*q2)); + + // } + // printf("\n"); + //} + + + + //double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + //toc = toc / 1000000; + //printf(" the copy data used is %f************\n", toc);//float fps = 1 / toc; + // // const unsigned char *input = (const unsigned char *)(img.data); + + + + } + + void VarianceFilter::integralImag_extract(const Mat &src, Mat &sum, Mat &sqsum)//void VarianceFilter::integralImag_extract(const Mat &src) + { + + + int i, j; + + cv::Mat t_sum; + //cv::Mat sum; + cv::Mat t_sqsum; + //cv::Mat sqsum; + + int vlen = 4; cl_event events[2]; + int offset = 0; + int pre_invalid = 0; + int vcols = (pre_invalid + src.cols + vlen - 1) / vlen; + + int w = src.cols, h = src.rows; + int depth = src.depth() == CV_8U ? CV_32S : CV_64F; + int type = CV_MAKE_TYPE(depth, 1); + +#ifdef debug + printf("the type is %d\n", depth); +#endif + t_sum.create(src.cols, src.rows, type); + sum.create(h, w, type); + + t_sqsum.create(src.cols, src.rows, CV_32FC1); + sqsum.create(h, w, CV_32FC1); + + int sum_offset = 0; // sum.offset / vlen; + int sqsum_offset = 0; // sqsum.offset / vlen; + + + cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY, (src.cols) * (src.rows) * sizeof(uchar), NULL, NULL); + cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(int), NULL, NULL); + cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(float), NULL, NULL); + cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(int), NULL, NULL); + cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(float), NULL, NULL); + //use host ptr + //cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (src.cols) * (src.rows) * sizeof(uchar), (void *)src.data, NULL); + //cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)t_sum.data, NULL); + //cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR , (src.cols) * (src.rows) * sizeof(float), (void *)t_sqsum.data, NULL); + //cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)sum.data, NULL); + //cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(float), (void *)sqsum.data, NULL); + + int step; + int src_step, sum_step; + int t_sum_step, sqsum_step; + + //src_step = 1920; + //t_sum_step = 4320; + //sum_step = 7680; + //sqsum_step = 7680; + int rows = src.rows; + int cols = src.cols; + src_step = src.step; + t_sum_step = t_sum.step; + sum_step = sum.step; + sqsum_step = sqsum.step; + + //printf(""); + status = clEnqueueWriteBuffer(commandQueue, srcdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(uchar), src.data, 0, NULL, NULL); + + assert(status == CL_SUCCESS); + //if (status != CL_SUCCESS) + //{ + // cout << "Error: src.data kernel_intgegral_cols EnqueueNDRangeKernel!" << endl; + //} + + status = clSetKernelArg(kernel_intgegral_cols, 0, sizeof(cl_mem), (void *)&srcdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&src.data)); + status = clSetKernelArg(kernel_intgegral_cols, 1, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_cols, 2, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_cols, 3, sizeof(cl_int), (void *)&offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&offset)); + status = clSetKernelArg(kernel_intgegral_cols, 4, sizeof(cl_int), (void *)&pre_invalid);//args.push_back(make_pair(sizeof(cl_int), (void *)&pre_invalid)); + status = clSetKernelArg(kernel_intgegral_cols, 5, sizeof(cl_int), (void *)&rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.rows)); + status = clSetKernelArg(kernel_intgegral_cols, 6, sizeof(cl_int), (void *)&cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.cols)); + status = clSetKernelArg(kernel_intgegral_cols, 7, sizeof(cl_int), &src_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.step)); + status = clSetKernelArg(kernel_intgegral_cols, 8, sizeof(cl_int), &t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + + + + size_t gt[3] = { ((vcols + 1) / 2) * 256, 1, 1 }, lt[3] = { 256, 1, 1 }; + // //openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_cols", gt, lt, args, -1, depth); + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols, 1, NULL, gt, lt, 0, NULL, &events[0]); + status = clWaitForEvents(1, &events[0]); + assert(status == CL_SUCCESS); + // + // status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols, 1, NULL, gt, lt, 0, NULL, NULL); + // + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_cols EnqueueNDRangeKernel!" << endl; + //} + // status = clWaitForEvents(1, &events[0]); + + //if (status != CL_SUCCESS) + // { + // printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + // } + + /*status = clEnqueueReadBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueReadBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueWriteBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueWriteBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + */ + + //clReleaseMemObject(tsumdata); + //clReleaseMemObject(tsqsumdata); + + + // tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)t_sum.data, NULL); + // tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(float), (void *)t_sqsum.data, NULL); + + + + status = clSetKernelArg(kernel_intgegral_rows, 0, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_rows, 1, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows, 2, sizeof(cl_mem), (void *)&sumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sum.data)); + status = clSetKernelArg(kernel_intgegral_rows, 3, sizeof(cl_mem), (void *)&sqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows, 4, sizeof(cl_int), (void *)&t_sum.rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.rows)); + status = clSetKernelArg(kernel_intgegral_rows, 5, sizeof(cl_int), (void *)&t_sum.cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.cols)); + status = clSetKernelArg(kernel_intgegral_rows, 6, sizeof(cl_int), (void *)&t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + status = clSetKernelArg(kernel_intgegral_rows, 7, sizeof(cl_int), (void *)&sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum.step)); + status = clSetKernelArg(kernel_intgegral_rows, 8, sizeof(cl_int), (void *)&sqsum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum.step)); + status = clSetKernelArg(kernel_intgegral_rows, 9, sizeof(cl_int), (void *)&sum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum_offset)); + status = clSetKernelArg(kernel_intgegral_rows, 10, sizeof(cl_int), (void *)&sqsum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum_offset)); + + size_t gt2[3] = { t_sum.cols * 32, 1, 1 }, lt2[3] = { 256, 1, 1 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_rows, 1, NULL, gt2, lt2, 0, NULL, &events[1]);//openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_rows", gt2, lt2, args, -1, depth); + + + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_rows EnqueueNDRangeKernel!" << endl; + //} + status = clWaitForEvents(1, &events[1]); + assert(status == CL_SUCCESS); + //if (status != CL_SUCCESS) + //{ + // printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + //} + + //convert clMat to mat + status = clEnqueueReadBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueReadBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + + + //psum = (float*) &sqsum.data; + //for (i = 0; i (img.size()); + integralImg_squared = new IntegralImage(img.size()); +#endif +#if PrintTime_integral + double tic = cvGetTickCount(); +#endif +//#if clIntegral_m_1 +// //integralImg->oclcalcIntImg(img); +// //integralImg_squared->oclcalcIntImgSquare(img, true); +// +// oclMat temp; +// temp = img; +// jifentu2(temp, sum, sqsum); //(m+1,n+1)-->(m,n) + +#if clIntegral_m + //integralImag_opencv( img); // (m,n)-->(m,n) + integralImag_extract(img, iSumMat, fSqreSumMat); + //integralImag_extract(img); + //integralImg->calcIntImg(img); + //integralImg_squared->calcIntImg(img, true); +#elif CPU + integralImg->calcIntImg(img); + integralImg_squared->calcIntImg(img, true); +#endif + + // int size = img.size().height*img.size().width; + //for (int i = 0; i < size; i++) + //{ + //#if clIntegral_m + // int img_dat = (int)iSumMat.data[i]; + // float img_dat2 = (float)fSqreSumMat.data[i]; + // int temp3 = img_dat; + //#else + // int img_dat = (int)integralImg->data[i]; + // float img_dat2 = (float)integralImg_squared->data[i]; + // int temp3 = img_dat; + // + //#endif + // } + +#if PrintTime_integral + double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the clIntegral time used is %f************\n", toc);//float fps = 1 / toc; +#endif + + + } + + bool VarianceFilter::filter(int i) + { + if (!enabled) return true; + + //float bboxvar = calcVariance(windowOffsets + TLD_WINDOW_OFFSET_SIZE * i); + float bboxvar; + int *off = windowOffsets + TLD_WINDOW_OFFSET_SIZE * i; + // + /// + int *ii1 = integralImg->data; + float *ii2 = integralImg_squared->data; + + float mX = (ii1[off[3]] - ii1[off[2]] - ii1[off[1]] + ii1[off[0]]) / (float)off[5]; //Sum of Area divided by area + float mX2 = (ii2[off[3]] - ii2[off[2]] - ii2[off[1]] + ii2[off[0]]) / (float)off[5]; + bboxvar = mX2 - mX * mX; + + + detectionResult->variances[i] = bboxvar; + //test for i + //if (i == 1172655) + // printf("i==%d,bboxvar=%f\n", i, bboxvar); + if (bboxvar < minVar) + { + return false; + } + + return true; + } + + + + + + + bool VarianceFilter::clfilter(const Mat &img) + { + //printf("begin using VarianceFilter clFilter*************************\n"); + + //char *kernelName = "M:\\OpenTLD\\OpenTLD-master\\OpenTLD-master\\src\\libopentld\\tld\\varianceFilter.cl"; + char *kernelName = "..\\..\\..\\src\\libopentld\\kernel\\varianceFilter_kernel.cpp"; // it is conbined to ensemble classify. + + cl_event events[1]; + string sourceStr; + status = convertToString(kernelName, sourceStr); + const char *source = sourceStr.c_str(); + size_t sourceSize[] = { strlen(source) }; + program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); + + /*Step 6: Build program. */ + status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + //printf("mid using GPU*************************\n"); + if (status != CL_SUCCESS) + { + cout << "VarianceFilter Error: Getting platforms!" << endl; + return false; + } + + cl_mem oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), (void *)windowOffsets, NULL); + cl_mem oclbuffII = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (img.size().width)*(img.size().height) * sizeof(int), (void *)iSumMat.data, NULL); + cl_mem oclbuffIISqure = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (img.size().width)*(img.size().height) * sizeof(int), (void *)fSqreSumMat.data, NULL); + cl_mem oclbuffDetectionResultVarious = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->variances, NULL); + cl_mem oclbuffDetectionResultPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, NULL); + + cl_mem oclbuffDetectionwindowFlags = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(int), (void *)detectionResult->windowFlags, NULL); + + /*Step 8: Create kernel object */ + kernel = clCreateKernel(program, "varianceFilter", NULL); + + /*Step 9: Sets Kernel arguments.*/ + status = clSetKernelArg(kernel, 0, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + status = clSetKernelArg(kernel, 1, sizeof(cl_mem), (void *)&oclbuffII); + status = clSetKernelArg(kernel, 2, sizeof(cl_mem), (void *)&oclbuffIISqure); + status = clSetKernelArg(kernel, 3, sizeof(cl_mem), (void *)&oclbuffDetectionResultVarious); + status = clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *)&oclbuffDetectionResultPosteriors); + status = clSetKernelArg(kernel, 5, sizeof(int), (void *)&numWindows); + status = clSetKernelArg(kernel, 6, sizeof(float), (void *)&minVar); + status = clSetKernelArg(kernel, 7, sizeof(cl_mem), (void*)&oclbuffDetectionwindowFlags); + //status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&TLD_WINDOW_OFFSET_SIZE); + + /*Step 10: Running the kernel.*/ + printf("begore opencl kernel numWindows=%d\n", numWindows); + size_t global_work_size[1] = { numWindows }; + size_t local_work_size[1] = { 256 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, NULL, 0, NULL, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error:VarianceFilter EnqueueNDRangeKernel!" << endl; + //return false; + } + status = clWaitForEvents(1, &events[0]); + + if (status != CL_SUCCESS) + { + printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + + } + //cout << "o" << endl; + + status = clReleaseEvent(events[0]); + + //printf("end using GPU*************************\n"); + //for (int i = 0; i < numWindows; i++) + // if(detectionResult->windowFlags[i]==1) + // printf("detectionResult[%d] is %d\n", i,detectionResult->windowFlags[i]); + + /*Step 11: Read the cout put back to host memory.*/ + //status = clEnqueueReadBuffer(commandQueue, outputBuffer, CL_TRUE, 0, 12 * sizeof(char), output, 0, NULL, NULL); -bool VarianceFilter::filter(int i) -{ - if(!enabled) return true; - float bboxvar = calcVariance(windowOffsets + TLD_WINDOW_OFFSET_SIZE * i); - detectionResult->variances[i] = bboxvar; + clReleaseKernel(kernel); + clReleaseProgram(program); - if(bboxvar < minVar) - { - return false; - } + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbuffII); + clReleaseMemObject(oclbuffIISqure); + clReleaseMemObject(oclbuffDetectionResultVarious); + clReleaseMemObject(oclbuffDetectionResultPosteriors); + clReleaseMemObject(oclbuffDetectionwindowFlags); + return true; - return true; -} + } } /* namespace tld */ diff --git a/src/libopentld/tld/VarianceFilter.h b/src/libopentld/tld/VarianceFilter.h index 19adda3e..31739f6d 100644 --- a/src/libopentld/tld/VarianceFilter.h +++ b/src/libopentld/tld/VarianceFilter.h @@ -17,44 +17,81 @@ * */ /* - * VarianceFilter.h - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* VarianceFilter.h +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #ifndef VARIANCEFILTER_H_ #define VARIANCEFILTER_H_ #include +#include #include "IntegralImage.h" #include "DetectionResult.h" +#include "switch.h" namespace tld { -class VarianceFilter -{ - IntegralImage* integralImg; - IntegralImage* integralImg_squared; + class VarianceFilter + { + + + public: + oclMat clsum, clsqsum; + + + //move here from top, which was set public from private. //edited by xqc + IntegralImage* integralImg; + IntegralImage* integralImg_squared; + bool enabled; + int *windowOffsets; + + DetectionResult *detectionResult; + + float minVar; + //for opencl begin + int numWindows; + cl_uint numPlatforms; //the NO. of platforms + cl_platform_id platform; //the chosen platform + cl_int status; + cl_uint numDevices; + + + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel kernel, kernel_intgegral_cols, kernel_intgegral_rows; + cv::Mat iSumMat; + cv::Mat fSqreSumMat; + + + + + //for opencl end + VarianceFilter(); + virtual ~VarianceFilter(); + + void release(); + void nextIteration(const cv::Mat &img); + bool filter(int idx); -public: - bool enabled; - int *windowOffsets; + bool clfilter(const Mat &img); + float calcVariance(int *off); + int VarianceFilter::convertToString(const char *filename, std::string& s); + void integralImag_opencv(const Mat &img); - DetectionResult *detectionResult; + void integralImag_extract(const Mat &src, Mat &iSumMat, Mat &fSqreSumMat); - float minVar; - VarianceFilter(); - virtual ~VarianceFilter(); - void release(); - void nextIteration(const cv::Mat &img); - bool filter(int idx); - float calcVariance(int *off); -}; + }; } /* namespace tld */ #endif /* VARIANCEFILTER_H_ */ diff --git a/src/libopentld/tld/switch.h b/src/libopentld/tld/switch.h new file mode 100644 index 00000000..bd49f27e --- /dev/null +++ b/src/libopentld/tld/switch.h @@ -0,0 +1,20 @@ +#define opencl 0 +#if opencl +#define CPU 0 +#else +#define CPU 1 +#endif + +#define clDetect 0 +#define clIntegral 0 +#define clIntegral_m 0 +#define clIntegral_m_1 0 +#define clOverlap 0 +#define clNNClassifier 0 + +#define PrintTime_overlap 0 +#define PrintTime_detect 0 +#define PrintTime_learn 0 +#define PrintTime_integral 0 +#define PrintTime_fuseHypotheses 0 +#define SELECTED_DEVICE_ID 0 \ No newline at end of file diff --git a/src/opentld/CMakeLists.txt b/src/opentld/CMakeLists.txt index 23170b33..dd20a37d 100644 --- a/src/opentld/CMakeLists.txt +++ b/src/opentld/CMakeLists.txt @@ -1,12 +1,14 @@ -link_directories(${OpenCV_LIB_DIR}) +link_directories(${OpenCV_LIB_DIR} +${OpenCL_LIB_DIR}) include_directories(main ../libopentld/imacq ../libopentld/mftracker ../libopentld/tld ../3rdparty/cvblobs - ${OpenCV_INCLUDE_DIRS}) + ${OpenCV_INCLUDE_DIRS} + ${OpenCL_INCLUDE_DIRS}) if(NOT USE_SYSTEM_LIBS) include_directories(../3rdparty/libconfig) @@ -26,14 +28,14 @@ add_library(main main/Settings.h main/Trajectory.h) -target_link_libraries(main libopentld cvblobs config++ ${OpenCV_LIBS}) +target_link_libraries(main libopentld cvblobs config++ ${OpenCV_LIBS} ${OPENCL_LIBRARIES}) #------------------------------------------------------------------------------- # opentld add_executable(opentld OpenTLD.cpp) -target_link_libraries(opentld main libopentld cvblobs config++ ${OpenCV_LIBS}) +target_link_libraries(opentld main libopentld cvblobs config++ ${OpenCV_LIBS} ${OPENCL_LIBRARIES}) install(TARGETS opentld DESTINATION bin) @@ -57,7 +59,7 @@ if(BUILD_QOPENTLD) set(QOPENTLD_SOURCES ${QOPENTLD_SOURCES} ${QOPENTLD_RCS_CPP} ${QOPENTLD_HEADERS_MOC} ${QOPENTLD_FORMS_HEADERS}) add_executable(qopentld ${QOPENTLD_SOURCES}) - target_link_libraries(qopentld main libopentld cvblobs config++ ${OpenCV_LIBS} ${QT_LIBRARIES}) + target_link_libraries(qopentld main libopentld cvblobs config++ ${OpenCV_LIBS ${OPENCL_LIBRARIES}} ${QT_LIBRARIES}) install(TARGETS qopentld DESTINATION bin) endif(BUILD_QOPENTLD) diff --git a/src/opentld/OpenTLD.cpp b/src/opentld/OpenTLD.cpp index 554f32ed..9a936ec5 100644 --- a/src/opentld/OpenTLD.cpp +++ b/src/opentld/OpenTLD.cpp @@ -18,51 +18,107 @@ */ /** - * @author Georg Nebehay - */ +* @author Georg Nebehay +*/ #include "Main.h" #include "Config.h" #include "ImAcq.h" #include "Gui.h" + + + + + using tld::Config; using tld::Gui; using tld::Settings; +#define __NO_STD_VECTOR +#define __CL_ENABLE_EXCEPTIONS + +#include +#include +#include +using namespace std; + + + +#include +#include +#include +#include +#include + +#define SUCCESS 0 +#define FAILURE 1 + +int convertToString(const char *filename, std::string& s) +{ + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); + + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } + + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return FAILURE; +} int main(int argc, char **argv) { - Main *main = new Main(); - Config config; - ImAcq *imAcq = imAcqAlloc(); - Gui *gui = new Gui(); - main->gui = gui; - main->imAcq = imAcq; - if(config.init(argc, argv) == PROGRAM_EXIT) - { - return EXIT_FAILURE; - } + Main *main = new Main(); + Config config; + ImAcq *imAcq = imAcqAlloc(); + Gui *gui = new Gui(); + + main->gui = gui; + main->imAcq = imAcq; + + if (config.init(argc, argv) == PROGRAM_EXIT) + { + return EXIT_FAILURE; + } - config.configure(main); + config.configure(main); - srand(main->seed); + srand(main->seed); - imAcqInit(imAcq); + imAcqInit(imAcq); - if(main->showOutput) - { - gui->init(); - } + if (main->showOutput) + { + gui->init(); + } main->doWork(); - main->doWork(); + delete main; + main = NULL; + delete gui; + gui = NULL; - delete main; - main = NULL; - delete gui; - gui = NULL; - return EXIT_SUCCESS; + return EXIT_SUCCESS; } + + diff --git a/src/opentld/main/Config.cpp b/src/opentld/main/Config.cpp index faec384c..0c777466 100644 --- a/src/opentld/main/Config.cpp +++ b/src/opentld/main/Config.cpp @@ -26,472 +26,472 @@ using namespace std; namespace tld { -static char help_text[] = - "usage: tld [option arguments] [arguments]\n" - "option arguments:\n" - "[-a ] video starts at the frameNumber \n" - "[-b ] Initial bounding box\n" - "[-c] shows color images instead of greyscale\n" - "[-d ] select input device: =(IMGS|CAM|VID)\n" - " IMGS: capture from images\n" - " CAM: capture from connected camera\n" - " VID: capture from a video\n" - " STREAM: capture from a stream\n" - "[-e ] export model after run to \n" - "[-f] shows foreground\n" - "[-i ] to the images or to the video\n" - "[-h] shows help\n" - "[-j ] specifies the of the last frames which are considered by the trajectory; 0 disables the trajectory\n" - "[-m ] if specified load a model from . An initialBoundingBox must be specified or selectManually must be true.\n" - "[-n ] specifies which camera device to use.\n" - "[-p ] prints results into the file \n" - "[-s] if set, user can select initial bounding box\n" - "[-t ] threshold for determining positive results\n" - "[-z ] video ends at the frameNumber .\n" - " If is 0 or the option argument isn't specified means\n" - " take all frames.\n" - "arguments:\n" - "[] to the config file\n"; - -Config::Config() : - m_selectManuallySet(false), - m_methodSet(false), - m_startFrameSet(false), - m_lastFrameSet(false), - m_trajectorySet(false), - m_showDetectionsSet(false), - m_showForegroundSet(false), - m_thetaSet(false), - m_printResultsSet(false), - m_camNoSet(false), - m_imagePathSet(false), - m_modelPathSet(false), - m_initialBBSet(false), - m_showOutputSet(false), - m_exportModelAfterRunSet(false) -{ -} - -Config::Config(Settings &settings) : - m_settings(settings) -{ -} - -Config::~Config() -{ -} - -int Config::init(int argc, char **argv) -{ - // check cli arguments - int c; - - while((c = getopt(argc, argv, "a:b:d:e:fhi:j:m:n:Op:qst:z:")) != -1) - { - switch(c) - { - case 'a': - m_settings.m_startFrame = atoi(optarg); - m_startFrameSet = true; - break; - case 'b': - char *pch; - pch = strtok(optarg, ","); - - while(pch != NULL) - { - m_settings.m_initialBoundingBox.push_back(atoi(pch)); - pch = strtok(NULL, ","); - } - - break; - case 'd': - - if(!strcmp(optarg, "CAM")) - { - m_settings.m_method = IMACQ_CAM; - m_methodSet = true; - } - else if(!strcmp(optarg, "VID")) - { - m_settings.m_method = IMACQ_VID; - m_methodSet = true; - } - else if(!strcmp(optarg, "IMGS")) - { - m_settings.m_method = IMACQ_IMGS; - m_methodSet = true; - } - else if(!strcmp(optarg, "STREAM")) - { - m_settings.m_method = IMACQ_STREAM; - m_methodSet = true; - } - - break; - case 'e': - m_settings.m_exportModelAfterRun = true; - m_settings.m_modelExportFile = optarg; - break; - case 'f': - m_settings.m_showForeground = true; - m_showForegroundSet = true; - break; - case 'h': - cout << help_text; - return PROGRAM_EXIT; - break; - case 'i': - m_settings.m_imagePath = optarg; - m_imagePathSet = true; - break; - case 'j': - m_settings.m_trajectory = atoi(optarg); - m_trajectorySet = true; - break; - case 'm': - m_settings.m_loadModel = true; - m_settings.m_modelPath = optarg; - m_modelPathSet = true; - break; - case 'n': - m_settings.m_camNo = atoi(optarg); - m_camNoSet = true; - break; - case 'p': - m_settings.m_printResults = optarg; - m_printResultsSet = true; - break; - case 'O': - m_settings.m_showOutput = false; - m_showOutputSet = true; - break; - case 's': - m_settings.m_selectManually = true; - m_selectManuallySet = true; - break; - case 't': - m_settings.m_threshold = atof(optarg); - m_thetaSet = true; - break; - case 'z': - m_settings.m_lastFrame = atoi(optarg); - m_lastFrameSet = true; - break; - } - } - - if(!m_imagePathSet && m_methodSet && (m_settings.m_method == IMACQ_VID || m_settings.m_method == IMACQ_IMGS)) - { - cerr << "Error: Must set imagePath and method if capturing from images or a video." << endl; - return PROGRAM_EXIT; - } - - if(argc > optind) - m_configPath = argv[optind]; - - - // load config file - if(!m_configPath.empty()) - { - // read config file - try - { - m_cfg.readFile(m_configPath.c_str()); - } - catch(const libconfig::FileIOException &fioex) - { - cerr << "I/O error while reading config file." << endl; - return PROGRAM_EXIT; - } - catch(const libconfig::ParseException &pex) - { - cerr << "ConfigFile: Parse error" << endl; - return PROGRAM_EXIT; - } - - // method - string method; - m_cfg.lookupValue("acq.method", method); - - // imagePath - if(method.compare("IMGS") == 0) - { - m_settings.m_method = IMACQ_IMGS; - - try - { - m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); - } - catch(const libconfig::SettingNotFoundException &nfex) - { - cerr << "Error: Unable to read image path." << endl; - return PROGRAM_EXIT; - } - } - else if(method.compare("STREAM") == 0) - { - m_settings.m_method = IMACQ_STREAM; - - try - { - m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); - } - catch(const libconfig::SettingNotFoundException &nfex) - { - cerr << "Error: Unable to read stream URL." << endl; - return PROGRAM_EXIT; - } - } - else if(method.compare("VID") == 0) - { - m_settings.m_method = IMACQ_VID; - - try - { - m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); - } - catch(const libconfig::SettingNotFoundException &nfex) - { - cerr << "Error: Unable to read image path." << endl; - return PROGRAM_EXIT; - } - } - else if(method.compare("CAM") == 0) - { - m_settings.m_method = IMACQ_CAM; - } - else if(method.compare("LIVESIM") == 0) - { - m_settings.m_method = IMACQ_LIVESIM; - //fps - m_cfg.lookupValue("acq.fps", m_settings.m_fps); - - try - { - m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); - } - catch(const libconfig::SettingNotFoundException &nfex) - { - cerr << "Error: Unable to read image path." << endl; - return PROGRAM_EXIT; - } - - - } - - // startFrame - if(!m_startFrameSet) - m_cfg.lookupValue("acq.startFrame", m_settings.m_startFrame); - - // lastFrame - if(!m_lastFrameSet) - m_cfg.lookupValue("acq.lastFrame", m_settings.m_lastFrame); - - // camNo - if(!m_camNoSet) - m_cfg.lookupValue("acq.camNo", m_settings.m_camNo); - - // loadModel - if(!m_modelPathSet) - m_cfg.lookupValue("loadModel", m_settings.m_loadModel); - - // modelPath - if(!m_modelPathSet) - m_cfg.lookupValue("modelPath", m_settings.m_modelPath); - - // check if loadModel and modelPath are set, if one of them is set - if(!m_modelPathSet) - if(m_settings.m_loadModel && m_settings.m_modelPath.empty()) - { - cerr << "Error: modelPath must be set when using loadModel" << endl; - return PROGRAM_EXIT; - } - - // useProportionalShift - m_cfg.lookupValue("detector.useProportionalShift", m_settings.m_useProportionalShift); - - // proportionalShift - m_cfg.lookupValue("detector.proportionalShift", m_settings.m_proportionalShift); - - // minScale - m_cfg.lookupValue("detector.minScale", m_settings.m_minScale); - - // maxScale - m_cfg.lookupValue("detector.maxScale", m_settings.m_maxScale); - - // minSize - m_cfg.lookupValue("detector.minSize", m_settings.m_minSize); - - // numTrees - m_cfg.lookupValue("detector.numTrees", m_settings.m_numTrees); - - // numFeatures - m_cfg.lookupValue("detector.numFeatures", m_settings.m_numFeatures); - - // numFeatures - m_cfg.lookupValue("detector.thetaP", m_settings.m_thetaP); - m_cfg.lookupValue("detector.thetaN", m_settings.m_thetaN); - - // backgroundFrame - // TODO - //const char * backgroundFrame = NULL; - //config_lookup_string(&m_config, "backgroundFrame", &backgroundFrame); - //if(backgroundFrame != NULL) { - // classifier->background = imAcqLoadImg(imAcq, (char *)backgroundFrame, CV_LOAD_IMAGE_GRAYSCALE); - //} - - // showOutput - if(!m_showOutputSet) - m_cfg.lookupValue("showOutput", m_settings.m_showOutput); - - // trajectory - if(!m_trajectorySet) - m_cfg.lookupValue("trajectory", m_settings.m_trajectory); - - // printResults - if(!m_printResultsSet) - m_cfg.lookupValue("printResults", m_settings.m_printResults); - - // printTiming - m_cfg.lookupValue("printTiming", m_settings.m_printTiming); - - // learningEnabled - m_cfg.lookupValue("learningEnabled", m_settings.m_learningEnabled); - - // trackerEnabled - m_cfg.lookupValue("trackerEnabled", m_settings.m_trackerEnabled); - - // varianceFilterEnabled - m_cfg.lookupValue("detector.varianceFilterEnabled", m_settings.m_varianceFilterEnabled); - - // emnsembleClassifierEnabled - m_cfg.lookupValue("detector.ensembleClassifierEnabled", m_settings.m_ensembleClassifierEnabled); - - // nnClassifierEnabled - m_cfg.lookupValue("detector.nnClassifierEnabled", m_settings.m_nnClassifierEnabled); - - // selectManually - if(!m_selectManuallySet) - m_cfg.lookupValue("selectManually", m_settings.m_selectManually); - - // saveDir - m_cfg.lookupValue("saveDir", m_settings.m_outputDir); - - // theta - if(!m_thetaSet) - m_cfg.lookupValue("threshold", m_settings.m_threshold); - - // showNotConfident - m_cfg.lookupValue("showNotConfident", m_settings.m_showNotConfident); - - // showForeground - if(!m_showForegroundSet) - m_cfg.lookupValue("showForeground", m_settings.m_showForeground); - - // showDetections - if(!m_showDetectionsSet) - m_cfg.lookupValue("showDetections", m_settings.m_showDetections); - - // alternating - m_cfg.lookupValue("alternating", m_settings.m_alternating); - - // exportModelFile - m_cfg.lookupValue("modelExportFile", m_settings.m_modelExportFile); - - // exportModelAfterRun - if(!m_exportModelAfterRunSet) - m_cfg.lookupValue("exportModelAfterRun", m_settings.m_exportModelAfterRun); - - // seed - m_cfg.lookupValue("seed", m_settings.m_seed); - - // initialBoundingBox - try - { - libconfig::Setting &initBB_setting = m_cfg.lookup("initialBoundingBox"); - - for(int i = 0; i < 4; i++) - { - m_settings.m_initialBoundingBox.push_back(initBB_setting[i]); - } - } - catch(const libconfig::SettingNotFoundException &nfex) - { - // Ignore - } - } - - return SUCCESS; -} - -int Config::configure(Main *main) -{ - ImAcq *imAcq = main->imAcq; - - // imAcq - imAcq->method = m_settings.m_method; - imAcq->imgPath = (m_settings.m_imagePath.empty()) ? NULL : m_settings.m_imagePath.c_str(); - imAcq->lastFrame = m_settings.m_lastFrame; - imAcq->currentFrame = m_settings.m_startFrame; - imAcq->camNo = m_settings.m_camNo; - imAcq->fps = m_settings.m_fps; - - // main - main->tld->trackerEnabled = m_settings.m_trackerEnabled; - main->showOutput = m_settings.m_showOutput; - main->showTrajectory = (m_settings.m_trajectory) ? true : false; - main->trajectoryLength = m_settings.m_trajectory; - main->printResults = (m_settings.m_printResults.empty()) ? NULL : m_settings.m_printResults.c_str(); - main->saveDir = (m_settings.m_outputDir.empty()) ? NULL : m_settings.m_outputDir.c_str(); - main->threshold = m_settings.m_threshold; - main->showForeground = m_settings.m_showForeground; - main->showNotConfident = m_settings.m_showNotConfident; - main->tld->alternating = m_settings.m_alternating; - main->tld->learningEnabled = m_settings.m_learningEnabled; - main->selectManually = m_settings.m_selectManually; - main->exportModelAfterRun = m_settings.m_exportModelAfterRun; - main->modelExportFile = m_settings.m_modelExportFile.c_str(); - main->loadModel = m_settings.m_loadModel; - main->modelPath = (m_settings.m_modelPath.empty()) ? NULL : m_settings.m_modelPath.c_str(); - main->seed = m_settings.m_seed; - - if(m_settings.m_initialBoundingBox.size() > 0) - { - main->initialBB = new int[4]; - - for(int i = 0; i < 4; i++) - { - main->initialBB[i] = m_settings.m_initialBoundingBox[i]; - } - } - - DetectorCascade *detectorCascade = main->tld->detectorCascade; - detectorCascade->varianceFilter->enabled = m_settings.m_varianceFilterEnabled; - detectorCascade->ensembleClassifier->enabled = m_settings.m_ensembleClassifierEnabled; - detectorCascade->nnClassifier->enabled = m_settings.m_nnClassifierEnabled; - - // classifier - detectorCascade->useShift = m_settings.m_useProportionalShift; - detectorCascade->shift = m_settings.m_proportionalShift; - detectorCascade->minScale = m_settings.m_minScale; - detectorCascade->maxScale = m_settings.m_maxScale; - detectorCascade->minSize = m_settings.m_minSize; - detectorCascade->numTrees = m_settings.m_numTrees; - detectorCascade->numFeatures = m_settings.m_numFeatures; - detectorCascade->nnClassifier->thetaTP = m_settings.m_thetaP; - detectorCascade->nnClassifier->thetaFP = m_settings.m_thetaN; - - return SUCCESS; -} - -/* - POSIX getopt for Windows - - AT&T Public License - - Code given out at the 1985 UNIFORUM conference in Dallas. - */ + static char help_text[] = + "usage: tld [option arguments] [arguments]\n" + "option arguments:\n" + "[-a ] video starts at the frameNumber \n" + "[-b ] Initial bounding box\n" + "[-c] shows color images instead of greyscale\n" + "[-d ] select input device: =(IMGS|CAM|VID)\n" + " IMGS: capture from images\n" + " CAM: capture from connected camera\n" + " VID: capture from a video\n" + " STREAM: capture from a stream\n" + "[-e ] export model after run to \n" + "[-f] shows foreground\n" + "[-i ] to the images or to the video\n" + "[-h] shows help\n" + "[-j ] specifies the of the last frames which are considered by the trajectory; 0 disables the trajectory\n" + "[-m ] if specified load a model from . An initialBoundingBox must be specified or selectManually must be true.\n" + "[-n ] specifies which camera device to use.\n" + "[-p ] prints results into the file \n" + "[-s] if set, user can select initial bounding box\n" + "[-t ] threshold for determining positive results\n" + "[-z ] video ends at the frameNumber .\n" + " If is 0 or the option argument isn't specified means\n" + " take all frames.\n" + "arguments:\n" + "[] to the config file\n"; + + Config::Config() : + m_selectManuallySet(false), + m_methodSet(false), + m_startFrameSet(false), + m_lastFrameSet(false), + m_trajectorySet(false), + m_showDetectionsSet(false), + m_showForegroundSet(false), + m_thetaSet(false), + m_printResultsSet(false), + m_camNoSet(false), + m_imagePathSet(false), + m_modelPathSet(false), + m_initialBBSet(false), + m_showOutputSet(false), + m_exportModelAfterRunSet(false) + { + } + + Config::Config(Settings &settings) : + m_settings(settings) + { + } + + Config::~Config() + { + } + + int Config::init(int argc, char **argv) + { + // check cli arguments + int c; + + while ((c = getopt(argc, argv, "a:b:d:e:fhi:j:m:n:Op:qst:z:")) != -1) + { + switch (c) + { + case 'a': + m_settings.m_startFrame = atoi(optarg); + m_startFrameSet = true; + break; + case 'b': + char *pch; + pch = strtok(optarg, ","); + + while (pch != NULL) + { + m_settings.m_initialBoundingBox.push_back(atoi(pch)); + pch = strtok(NULL, ","); + } + + break; + case 'd': + + if (!strcmp(optarg, "CAM")) + { + m_settings.m_method = IMACQ_CAM; + m_methodSet = true; + } + else if (!strcmp(optarg, "VID")) + { + m_settings.m_method = IMACQ_VID; + m_methodSet = true; + } + else if (!strcmp(optarg, "IMGS")) + { + m_settings.m_method = IMACQ_IMGS; + m_methodSet = true; + } + else if (!strcmp(optarg, "STREAM")) + { + m_settings.m_method = IMACQ_STREAM; + m_methodSet = true; + } + + break; + case 'e': + m_settings.m_exportModelAfterRun = true; + m_settings.m_modelExportFile = optarg; + break; + case 'f': + m_settings.m_showForeground = true; + m_showForegroundSet = true; + break; + case 'h': + cout << help_text; + return PROGRAM_EXIT; + break; + case 'i': + m_settings.m_imagePath = optarg; + m_imagePathSet = true; + break; + case 'j': + m_settings.m_trajectory = atoi(optarg); + m_trajectorySet = true; + break; + case 'm': + m_settings.m_loadModel = true; + m_settings.m_modelPath = optarg; + m_modelPathSet = true; + break; + case 'n': + m_settings.m_camNo = atoi(optarg); + m_camNoSet = true; + break; + case 'p': + m_settings.m_printResults = optarg; + m_printResultsSet = true; + break; + case 'O': + m_settings.m_showOutput = false; + m_showOutputSet = true; + break; + case 's': + m_settings.m_selectManually = true; + m_selectManuallySet = true; + break; + case 't': + m_settings.m_threshold = atof(optarg); + m_thetaSet = true; + break; + case 'z': + m_settings.m_lastFrame = atoi(optarg); + m_lastFrameSet = true; + break; + } + } + + if (!m_imagePathSet && m_methodSet && (m_settings.m_method == IMACQ_VID || m_settings.m_method == IMACQ_IMGS)) + { + cerr << "Error: Must set imagePath and method if capturing from images or a video." << endl; + return PROGRAM_EXIT; + } + + if (argc > optind) + m_configPath = argv[optind]; + + + // load config file + if (!m_configPath.empty()) + { + // read config file + try + { + m_cfg.readFile(m_configPath.c_str()); + } + catch (const libconfig::FileIOException &fioex) + { + cerr << "I/O error while reading config file." << endl; + return PROGRAM_EXIT; + } + catch (const libconfig::ParseException &pex) + { + cerr << "ConfigFile: Parse error" << endl; + return PROGRAM_EXIT; + } + + // method + string method; + m_cfg.lookupValue("acq.method", method); + + // imagePath + if (method.compare("IMGS") == 0) + { + m_settings.m_method = IMACQ_IMGS; + + try + { + m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); + } + catch (const libconfig::SettingNotFoundException &nfex) + { + cerr << "Error: Unable to read image path." << endl; + return PROGRAM_EXIT; + } + } + else if (method.compare("STREAM") == 0) + { + m_settings.m_method = IMACQ_STREAM; + + try + { + m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); + } + catch (const libconfig::SettingNotFoundException &nfex) + { + cerr << "Error: Unable to read stream URL." << endl; + return PROGRAM_EXIT; + } + } + else if (method.compare("VID") == 0) + { + m_settings.m_method = IMACQ_VID; + + try + { + m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); + } + catch (const libconfig::SettingNotFoundException &nfex) + { + cerr << "Error: Unable to read image path." << endl; + return PROGRAM_EXIT; + } + } + else if (method.compare("CAM") == 0) + { + m_settings.m_method = IMACQ_CAM; + } + else if (method.compare("LIVESIM") == 0) + { + m_settings.m_method = IMACQ_LIVESIM; + //fps + m_cfg.lookupValue("acq.fps", m_settings.m_fps); + + try + { + m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); + } + catch (const libconfig::SettingNotFoundException &nfex) + { + cerr << "Error: Unable to read image path." << endl; + return PROGRAM_EXIT; + } + + + } + + // startFrame + if (!m_startFrameSet) + m_cfg.lookupValue("acq.startFrame", m_settings.m_startFrame); + + // lastFrame + if (!m_lastFrameSet) + m_cfg.lookupValue("acq.lastFrame", m_settings.m_lastFrame); + + // camNo + if (!m_camNoSet) + m_cfg.lookupValue("acq.camNo", m_settings.m_camNo); + + // loadModel + if (!m_modelPathSet) + m_cfg.lookupValue("loadModel", m_settings.m_loadModel); + + // modelPath + if (!m_modelPathSet) + m_cfg.lookupValue("modelPath", m_settings.m_modelPath); + + // check if loadModel and modelPath are set, if one of them is set + if (!m_modelPathSet) + if (m_settings.m_loadModel && m_settings.m_modelPath.empty()) + { + cerr << "Error: modelPath must be set when using loadModel" << endl; + return PROGRAM_EXIT; + } + + // useProportionalShift + m_cfg.lookupValue("detector.useProportionalShift", m_settings.m_useProportionalShift); + + // proportionalShift + m_cfg.lookupValue("detector.proportionalShift", m_settings.m_proportionalShift); + + // minScale + m_cfg.lookupValue("detector.minScale", m_settings.m_minScale); + + // maxScale + m_cfg.lookupValue("detector.maxScale", m_settings.m_maxScale); + + // minSize + m_cfg.lookupValue("detector.minSize", m_settings.m_minSize); + + // numTrees + m_cfg.lookupValue("detector.numTrees", m_settings.m_numTrees); + + // numFeatures + m_cfg.lookupValue("detector.numFeatures", m_settings.m_numFeatures); + + // numFeatures + m_cfg.lookupValue("detector.thetaP", m_settings.m_thetaP); + m_cfg.lookupValue("detector.thetaN", m_settings.m_thetaN); + + // backgroundFrame + // TODO + //const char * backgroundFrame = NULL; + //config_lookup_string(&m_config, "backgroundFrame", &backgroundFrame); + //if(backgroundFrame != NULL) { + // classifier->background = imAcqLoadImg(imAcq, (char *)backgroundFrame, CV_LOAD_IMAGE_GRAYSCALE); + //} + + // showOutput + if (!m_showOutputSet) + m_cfg.lookupValue("showOutput", m_settings.m_showOutput); + + // trajectory + if (!m_trajectorySet) + m_cfg.lookupValue("trajectory", m_settings.m_trajectory); + + // printResults + if (!m_printResultsSet) + m_cfg.lookupValue("printResults", m_settings.m_printResults); + + // printTiming + m_cfg.lookupValue("printTiming", m_settings.m_printTiming); + + // learningEnabled + m_cfg.lookupValue("learningEnabled", m_settings.m_learningEnabled); + + // trackerEnabled + m_cfg.lookupValue("trackerEnabled", m_settings.m_trackerEnabled); + + // varianceFilterEnabled + m_cfg.lookupValue("detector.varianceFilterEnabled", m_settings.m_varianceFilterEnabled); + + // emnsembleClassifierEnabled + m_cfg.lookupValue("detector.ensembleClassifierEnabled", m_settings.m_ensembleClassifierEnabled); + + // nnClassifierEnabled + m_cfg.lookupValue("detector.nnClassifierEnabled", m_settings.m_nnClassifierEnabled); + + // selectManually + if (!m_selectManuallySet) + m_cfg.lookupValue("selectManually", m_settings.m_selectManually); + + // saveDir + m_cfg.lookupValue("saveDir", m_settings.m_outputDir); + + // theta + if (!m_thetaSet) + m_cfg.lookupValue("threshold", m_settings.m_threshold); + + // showNotConfident + m_cfg.lookupValue("showNotConfident", m_settings.m_showNotConfident); + + // showForeground + if (!m_showForegroundSet) + m_cfg.lookupValue("showForeground", m_settings.m_showForeground); + + // showDetections + if (!m_showDetectionsSet) + m_cfg.lookupValue("showDetections", m_settings.m_showDetections); + + // alternating + m_cfg.lookupValue("alternating", m_settings.m_alternating); + + // exportModelFile + m_cfg.lookupValue("modelExportFile", m_settings.m_modelExportFile); + + // exportModelAfterRun + if (!m_exportModelAfterRunSet) + m_cfg.lookupValue("exportModelAfterRun", m_settings.m_exportModelAfterRun); + + // seed + m_cfg.lookupValue("seed", m_settings.m_seed); + + // initialBoundingBox + try + { + libconfig::Setting &initBB_setting = m_cfg.lookup("initialBoundingBox"); + + for (int i = 0; i < 4; i++) + { + m_settings.m_initialBoundingBox.push_back(initBB_setting[i]); + } + } + catch (const libconfig::SettingNotFoundException &nfex) + { + // Ignore + } + } + + return SUCCESS; + } + + int Config::configure(Main *main) + { + ImAcq *imAcq = main->imAcq; + + // imAcq + imAcq->method = m_settings.m_method; + imAcq->imgPath = (m_settings.m_imagePath.empty()) ? NULL : m_settings.m_imagePath.c_str(); + imAcq->lastFrame = m_settings.m_lastFrame; + imAcq->currentFrame = m_settings.m_startFrame; + imAcq->camNo = m_settings.m_camNo; + imAcq->fps = m_settings.m_fps; + + // main + main->tld->trackerEnabled = m_settings.m_trackerEnabled; + main->showOutput = m_settings.m_showOutput; + main->showTrajectory = (m_settings.m_trajectory) ? true : false; + main->trajectoryLength = m_settings.m_trajectory; + main->printResults = (m_settings.m_printResults.empty()) ? NULL : m_settings.m_printResults.c_str(); + main->saveDir = (m_settings.m_outputDir.empty()) ? NULL : m_settings.m_outputDir.c_str(); + main->threshold = m_settings.m_threshold; + main->showForeground = m_settings.m_showForeground; + main->showNotConfident = m_settings.m_showNotConfident; + main->tld->alternating = m_settings.m_alternating; + main->tld->learningEnabled = m_settings.m_learningEnabled; + main->selectManually = m_settings.m_selectManually; + main->exportModelAfterRun = m_settings.m_exportModelAfterRun; + main->modelExportFile = m_settings.m_modelExportFile.c_str(); + main->loadModel = m_settings.m_loadModel; + main->modelPath = (m_settings.m_modelPath.empty()) ? NULL : m_settings.m_modelPath.c_str(); + main->seed = m_settings.m_seed; + + if (m_settings.m_initialBoundingBox.size() > 0) + { + main->initialBB = new int[4]; + + for (int i = 0; i < 4; i++) + { + main->initialBB[i] = m_settings.m_initialBoundingBox[i]; + } + } + + DetectorCascade *detectorCascade = main->tld->detectorCascade; + detectorCascade->varianceFilter->enabled = m_settings.m_varianceFilterEnabled; + detectorCascade->ensembleClassifier->enabled = m_settings.m_ensembleClassifierEnabled; + detectorCascade->nnClassifier->enabled = m_settings.m_nnClassifierEnabled; + + // classifier + detectorCascade->useShift = m_settings.m_useProportionalShift; + detectorCascade->shift = m_settings.m_proportionalShift; + detectorCascade->minScale = m_settings.m_minScale; + detectorCascade->maxScale = m_settings.m_maxScale; + detectorCascade->minSize = m_settings.m_minSize; + detectorCascade->numTrees = m_settings.m_numTrees; + detectorCascade->numFeatures = m_settings.m_numFeatures; + detectorCascade->nnClassifier->thetaTP = m_settings.m_thetaP; + detectorCascade->nnClassifier->thetaFP = m_settings.m_thetaN; + + return SUCCESS; + } + + /* + POSIX getopt for Windows + + AT&T Public License + + Code given out at the 1985 UNIFORUM conference in Dallas. + */ #ifndef __GNUC__ #define NULL 0 @@ -501,69 +501,69 @@ int Config::configure(Main *main) fputs(s, stderr);\ fputc(c, stderr);} -int opterr = 1; -int optind = 1; -int optopt; -char *optarg; - -int getopt(int argc, char **argv, char *opts) -{ - static int sp = 1; - register int c; - register char *cp; - - if(sp == 1) - if(optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') - return (EOF); - else if(strcmp(argv[optind], "--") == NULL) - { - optind++; - return (EOF); - } - - optopt = c = argv[optind][sp]; - - if(c == ':' || (cp = strchr(opts, c)) == NULL) - { - ERR(": illegal option -- ", c); - - if(argv[optind][++sp] == '\0') - { - optind++; - sp = 1; - } - - return ('?'); - } - - if(*++cp == ':') - { - if(argv[optind][sp + 1] != '\0') - optarg = &argv[optind++][sp + 1]; - else if(++optind >= argc) - { - ERR(": option requires an argument -- ", c); - sp = 1; - return ('?'); - } - else - optarg = argv[optind++]; - - sp = 1; - } - else - { - if(argv[optind][++sp] == '\0') - { - sp = 1; - optind++; - } - - optarg = NULL; - } - - return (c); -} + int opterr = 1; + int optind = 1; + int optopt; + char *optarg; + + int getopt(int argc, char **argv, char *opts) + { + static int sp = 1; + register int c; + register char *cp; + + if (sp == 1) + if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') + return (EOF); + else if (strcmp(argv[optind], "--") == NULL) + { + optind++; + return (EOF); + } + + optopt = c = argv[optind][sp]; + + if (c == ':' || (cp = strchr(opts, c)) == NULL) + { + ERR(": illegal option -- ", c); + + if (argv[optind][++sp] == '\0') + { + optind++; + sp = 1; + } + + return ('?'); + } + + if (*++cp == ':') + { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) + { + ERR(": option requires an argument -- ", c); + sp = 1; + return ('?'); + } + else + optarg = argv[optind++]; + + sp = 1; + } + else + { + if (argv[optind][++sp] == '\0') + { + sp = 1; + optind++; + } + + optarg = NULL; + } + + return (c); + } #endif /* __GNUC__ */ diff --git a/src/opentld/main/Gui.cpp b/src/opentld/main/Gui.cpp index 2f377286..a247c02c 100644 --- a/src/opentld/main/Gui.cpp +++ b/src/opentld/main/Gui.cpp @@ -18,15 +18,16 @@ */ /* - * gui.cpp - * - * Created on: Oct 18, 2011 - * Author: clemensk - */ +* gui.cpp +* +* Created on: Oct 18, 2011 +* Author: clemensk +*/ #include "Gui.h" #include "Main.h" +#include #include @@ -35,120 +36,151 @@ using std::string; namespace tld { -Gui::Gui() : - m_window_name("tld") -{ -} + Gui::Gui() : + m_window_name("tld") + { + } -Gui::~Gui() -{ -} + Gui::~Gui() + { + } -void Gui::init() -{ - cvNamedWindow(m_window_name.c_str(), CV_WINDOW_AUTOSIZE); - cvMoveWindow(m_window_name.c_str(), 100, 100); -} + void Gui::init() + { + cvNamedWindow(m_window_name.c_str(), CV_WINDOW_AUTOSIZE); + cvMoveWindow(m_window_name.c_str(), 100, 100); + } -void Gui::showImage(IplImage *image) -{ - cvShowImage(m_window_name.c_str(), image); -} + void Gui::showImage(IplImage *image) + { + cvShowImage(m_window_name.c_str(), image); + } + + char Gui::getKey() + { + return cvWaitKey(10); + } -char Gui::getKey() -{ - return cvWaitKey(10); -} + std::string Gui::windowName() + { + return m_window_name; + } -std::string Gui::windowName() -{ - return m_window_name; -} + static string window_name; + static CvFont font; + static IplImage *img0; + static IplImage *img1; + static CvPoint point; + static CvRect *bb; + static int drag = 0; + + static void mouseHandler(int event, int x, int y, int flags, void *param) + { + /* user press left button */ + if (event == CV_EVENT_LBUTTONDOWN && !drag) + { + point = cvPoint(x, y); + drag = 1; + } + + /* user drag the mouse */ + if (event == CV_EVENT_MOUSEMOVE && drag) + { + img1 = (IplImage *)cvClone(img0); + + cvRectangle(img1, point, cvPoint(x, y), CV_RGB(255, 0, 0), 1, 8, 0); + + cvShowImage(window_name.c_str(), img1); + cvReleaseImage(&img1); + } + + /* user release left button */ + if (event == CV_EVENT_LBUTTONUP && drag) + { + *bb = cvRect(point.x, point.y, x - point.x, y - point.y); + drag = 0; + } + } + + // TODO: member of Gui + // --> problem: callback function mouseHandler as member! + int getBBFromUser(IplImage *img, CvRect &rect, Gui *gui) + { + //IplImage* dst; + window_name = gui->windowName(); + img0 = (IplImage *)cvClone(img); + rect = cvRect(-1, -1, -1, -1); + bb = ▭ + bool correctBB = false; + cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, 0.5, 0.5, 0, 1, 8); + + cvSetMouseCallback(window_name.c_str(), mouseHandler, NULL); + cvPutText(img0, "Draw a bounding box and press Enter", cvPoint(0, 60), + &font, cvScalar(255, 255, 0)); + + + cvShowImage(window_name.c_str(), img0); + + + + //end dfafdads + + + + + + + while (!correctBB) + { + char key = cvWaitKey(0); + + if (tolower(key) == 'q') + { + return PROGRAM_EXIT; + } + + if (((key == '\n') || (key == '\r') || (key == '\r\n')) && (bb->x != -1) && (bb->y != -1)) + { + correctBB = true; + } + } + + if (rect.width < 0) + { + rect.x += rect.width; + rect.width = abs(rect.width); + } + + if (rect.height < 0) + { + rect.y += rect.height; + rect.height = abs(rect.height); + } + + cvSetMouseCallback(window_name.c_str(), NULL, NULL); + //cvShowImage(window_name.c_str(), img0); + + //dfafdads + //cvNamedWindow("源图像", 1); + //cvShowImage("源图像", img0); + //cvSetImageROI(img0, cvRect(100, 100, 0.5*img0->width, 0.5*img0->height)); + cvSetImageROI(img0, cvRect(rect.x, rect.y, rect.width, rect.height)); -static string window_name; -static CvFont font; -static IplImage *img0; -static IplImage *img1; -static CvPoint point; -static CvRect *bb; -static int drag = 0; + //dst = cvCreateImage(cvSize(rect.width,rect.height), IPL_DEPTH_8U, img0->nChannels); + //cvCopy(img0, dst, 0); + cvResetImageROI(img0); -static void mouseHandler(int event, int x, int y, int flags, void *param) -{ - /* user press left button */ - if(event == CV_EVENT_LBUTTONDOWN && !drag) - { - point = cvPoint(x, y); - drag = 1; - } - - /* user drag the mouse */ - if(event == CV_EVENT_MOUSEMOVE && drag) - { - img1 = (IplImage *) cvClone(img0); - - cvRectangle(img1, point, cvPoint(x, y), CV_RGB(255, 0, 0), 1, 8, 0); - - cvShowImage(window_name.c_str(), img1); - cvReleaseImage(&img1); - } - - /* user release left button */ - if(event == CV_EVENT_LBUTTONUP && drag) - { - *bb = cvRect(point.x, point.y, x - point.x, y - point.y); - drag = 0; - } -} + //cvNamedWindow("操作后的图像", 1); + //cvShowImage("操作后的图像", dst); + //cvDestroyWindow("操作后的图像"); + //cvDestroyWindow("源图像"); + //cvSaveImage("crop.jpg", dst); + //Sleep(1000); -// TODO: member of Gui -// --> problem: callback function mouseHandler as member! -int getBBFromUser(IplImage *img, CvRect &rect, Gui *gui) -{ - window_name = gui->windowName(); - img0 = (IplImage *) cvClone(img); - rect = cvRect(-1, -1, -1, -1); - bb = ▭ - bool correctBB = false; - cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, 0.5, 0.5, 0, 1, 8); - - cvSetMouseCallback(window_name.c_str(), mouseHandler, NULL); - cvPutText(img0, "Draw a bounding box and press Enter", cvPoint(0, 60), - &font, cvScalar(255, 255, 0)); - cvShowImage(window_name.c_str(), img0); - - while(!correctBB) - { - char key = cvWaitKey(0); - - if(tolower(key) == 'q') - { - return PROGRAM_EXIT; - } - - if(((key == '\n') || (key == '\r') || (key == '\r\n')) && (bb->x != -1) && (bb->y != -1)) - { - correctBB = true; - } - } - - if(rect.width < 0) - { - rect.x += rect.width; - rect.width = abs(rect.width); - } - - if(rect.height < 0) - { - rect.y += rect.height; - rect.height = abs(rect.height); - } - - cvSetMouseCallback(window_name.c_str(), NULL, NULL); - - cvReleaseImage(&img0); - cvReleaseImage(&img1); - - return SUCCESS; -} + //cvReleaseImage(&dst); + cvReleaseImage(&img0); + cvReleaseImage(&img1); + + return SUCCESS; + } } diff --git a/src/opentld/main/Main.cpp b/src/opentld/main/Main.cpp index d0584797..bba5dcd1 100644 --- a/src/opentld/main/Main.cpp +++ b/src/opentld/main/Main.cpp @@ -17,11 +17,11 @@ * */ /* - * MainX.cpp - * - * Created on: Nov 17, 2011 - * Author: Georg Nebehay - */ +* MainX.cpp +* +* Created on: Nov 17, 2011 +* Author: Georg Nebehay +*/ #include "Main.h" @@ -36,266 +36,279 @@ using namespace cv; void Main::doWork() { + IplImage* dst; Trajectory trajectory; - IplImage *img = imAcqGetImg(imAcq); - Mat grey(img->height, img->width, CV_8UC1); - cvtColor(cvarrToMat(img), grey, CV_BGR2GRAY); + IplImage *img = imAcqGetImg(imAcq); + Mat grey(img->height, img->width, CV_8UC1); + cvtColor(cvarrToMat(img), grey, CV_BGR2GRAY); - tld->detectorCascade->imgWidth = grey.cols; - tld->detectorCascade->imgHeight = grey.rows; - tld->detectorCascade->imgWidthStep = grey.step; + tld->detectorCascade->imgWidth = grey.cols; + tld->detectorCascade->imgHeight = grey.rows; + tld->detectorCascade->imgWidthStep = grey.step; - if(showTrajectory) + if (showTrajectory) { trajectory.init(trajectoryLength); } - if(selectManually) - { - - CvRect box; - - if(getBBFromUser(img, box, gui) == PROGRAM_EXIT) - { - return; - } - - if(initialBB == NULL) - { - initialBB = new int[4]; - } - - initialBB[0] = box.x; - initialBB[1] = box.y; - initialBB[2] = box.width; - initialBB[3] = box.height; - } - - FILE *resultsFile = NULL; - - if(printResults != NULL) - { - resultsFile = fopen(printResults, "w"); - if(!resultsFile) - { - fprintf(stderr, "Error: Unable to create results-file \"%s\"\n", printResults); - exit(-1); - } - } - - bool reuseFrameOnce = false; - bool skipProcessingOnce = false; - - if(loadModel && modelPath != NULL) - { - tld->readFromFile(modelPath); - reuseFrameOnce = true; - } - else if(initialBB != NULL) - { - Rect bb = tldArrayToRect(initialBB); - - printf("Starting at %d %d %d %d\n", bb.x, bb.y, bb.width, bb.height); - - tld->selectObject(grey, &bb); - skipProcessingOnce = true; - reuseFrameOnce = true; - } - - while(imAcqHasMoreFrames(imAcq)) - { - double tic = cvGetTickCount(); - - if(!reuseFrameOnce) - { - cvReleaseImage(&img); - img = imAcqGetImg(imAcq); - - if(img == NULL) - { - printf("current image is NULL, assuming end of input.\n"); - break; - } - - cvtColor(cvarrToMat(img), grey, CV_BGR2GRAY); - } - - if(!skipProcessingOnce) - { - tld->processImage(cvarrToMat(img)); - } - else - { - skipProcessingOnce = false; - } - - if(printResults != NULL) - { - if(tld->currBB != NULL) - { - fprintf(resultsFile, "%d %.2d %.2d %.2d %.2d %f\n", imAcq->currentFrame - 1, tld->currBB->x, tld->currBB->y, tld->currBB->width, tld->currBB->height, tld->currConf); - } - else - { - fprintf(resultsFile, "%d NaN NaN NaN NaN NaN\n", imAcq->currentFrame - 1); - } - } - - double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); - - toc = toc / 1000000; - - float fps = 1 / toc; - - int confident = (tld->currConf >= threshold) ? 1 : 0; - - if(showOutput || saveDir != NULL) - { - char string[128]; - - char learningString[10] = ""; - - if(tld->learning) - { - strcpy(learningString, "Learning"); - } - - sprintf(string, "#%d,Posterior %.2f; fps: %.2f, #numwindows:%d, %s", imAcq->currentFrame - 1, tld->currConf, fps, tld->detectorCascade->numWindows, learningString); - CvScalar yellow = CV_RGB(255, 255, 0); - CvScalar blue = CV_RGB(0, 0, 255); - CvScalar black = CV_RGB(0, 0, 0); - CvScalar white = CV_RGB(255, 255, 255); - - if(tld->currBB != NULL) - { - CvScalar rectangleColor = (confident) ? blue : yellow; - cvRectangle(img, tld->currBB->tl(), tld->currBB->br(), rectangleColor, 8, 8, 0); - - if(showTrajectory) + if (selectManually) + { + + CvRect box; + + if (getBBFromUser(img, box, gui) == PROGRAM_EXIT) + { + return; + } + + if (initialBB == NULL) + { + initialBB = new int[4]; + } + + initialBB[0] = box.x; + initialBB[1] = box.y; + initialBB[2] = box.width; + initialBB[3] = box.height; + //initialBB[0] = 650; + //initialBB[1] = 550; + //initialBB[2] = 300; + //initialBB[3] = 300; + //initialBB[0] = 120; + //initialBB[1] = 120; + //initialBB[2] = 110; + //initialBB[3] = 90; + + + + } + + FILE *resultsFile = NULL; + + if (printResults != NULL) + { + resultsFile = fopen(printResults, "w"); + if (!resultsFile) + { + fprintf(stderr, "Error: Unable to create results-file \"%s\"\n", printResults); + exit(-1); + } + } + + bool reuseFrameOnce = false; + bool skipProcessingOnce = false; + + if (loadModel && modelPath != NULL) + { + tld->readFromFile(modelPath); + reuseFrameOnce = true; + } + else if (initialBB != NULL) + { + Rect bb = tldArrayToRect(initialBB); + + printf("****************Starting at %d %d %d %d***********\n", bb.x, bb.y, bb.width, bb.height); + + tld->selectObject(grey, &bb); + skipProcessingOnce = true; + reuseFrameOnce = true; + } + + while (imAcqHasMoreFrames(imAcq)) + { + double tic = cvGetTickCount(); + + if (!reuseFrameOnce) + { + cvReleaseImage(&img); + img = imAcqGetImg(imAcq); + + if (img == NULL) + { + printf("current image is NULL, assuming end of input.\n"); + break; + } + + cvtColor(cvarrToMat(img), grey, CV_BGR2GRAY); + } + + if (!skipProcessingOnce) + { + tld->detectorCascade->FramNum = imAcq->currentFrame - 1; + tld->processImage(cvarrToMat(img)); + } + else + { + skipProcessingOnce = false; + } + + if (printResults != NULL) + { + if (tld->currBB != NULL) + { + fprintf(resultsFile, "%d %.2d %.2d %.2d %.2d %f\n", imAcq->currentFrame - 1, tld->currBB->x, tld->currBB->y, tld->currBB->width, tld->currBB->height, tld->currConf); + } + else + { + fprintf(resultsFile, "%d NaN NaN NaN NaN NaN\n", imAcq->currentFrame - 1); + } + } + + double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + + float fps = 1 / toc; + + int confident = (tld->currConf >= threshold) ? 1 : 0; + + if (showOutput || saveDir != NULL) + { + char string[128]; + + char learningString[10] = ""; + + if (tld->learning) + { + strcpy(learningString, "Learning"); + } + + sprintf(string, "#%d,Posterior %.2f; fps: %.2f, #numwindows:%d, %s", imAcq->currentFrame - 1, tld->currConf, fps, tld->detectorCascade->numWindows, learningString); + CvScalar yellow = CV_RGB(255, 255, 0); + CvScalar blue = CV_RGB(0, 0, 255); + CvScalar black = CV_RGB(0, 0, 0); + CvScalar white = CV_RGB(255, 255, 255); + + if (tld->currBB != NULL) + { + CvScalar rectangleColor = (confident) ? blue : yellow; + cvRectangle(img, tld->currBB->tl(), tld->currBB->br(), rectangleColor, 8, 8, 0); + + if (showTrajectory) { - CvPoint center = cvPoint(tld->currBB->x+tld->currBB->width/2, tld->currBB->y+tld->currBB->height/2); - cvLine(img, cvPoint(center.x-2, center.y-2), cvPoint(center.x+2, center.y+2), rectangleColor, 2); - cvLine(img, cvPoint(center.x-2, center.y+2), cvPoint(center.x+2, center.y-2), rectangleColor, 2); + CvPoint center = cvPoint(tld->currBB->x + tld->currBB->width / 2, tld->currBB->y + tld->currBB->height / 2); + cvLine(img, cvPoint(center.x - 2, center.y - 2), cvPoint(center.x + 2, center.y + 2), rectangleColor, 2); + cvLine(img, cvPoint(center.x - 2, center.y + 2), cvPoint(center.x + 2, center.y - 2), rectangleColor, 2); trajectory.addPoint(center, rectangleColor); } - } - else if(showTrajectory) + } + else if (showTrajectory) { trajectory.addPoint(cvPoint(-1, -1), cvScalar(-1, -1, -1)); } - if(showTrajectory) + if (showTrajectory) { trajectory.drawTrajectory(img); } - CvFont font; - cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, .5, .5, 0, 1, 8); - cvRectangle(img, cvPoint(0, 0), cvPoint(img->width, 50), black, CV_FILLED, 8, 0); - cvPutText(img, string, cvPoint(25, 25), &font, white); - - if(showForeground) - { - - for(size_t i = 0; i < tld->detectorCascade->detectionResult->fgList->size(); i++) - { - Rect r = tld->detectorCascade->detectionResult->fgList->at(i); - cvRectangle(img, r.tl(), r.br(), white, 1); - } - - } - - - if(showOutput) - { - gui->showImage(img); - char key = gui->getKey(); - - if(key == 'q') break; - - if(key == 'b') - { - - ForegroundDetector *fg = tld->detectorCascade->foregroundDetector; - - if(fg->bgImg.empty()) - { - fg->bgImg = grey.clone(); - } - else - { - fg->bgImg.release(); - } - } - - if(key == 'c') - { - //clear everything - tld->release(); - } - - if(key == 'l') - { - tld->learningEnabled = !tld->learningEnabled; - printf("LearningEnabled: %d\n", tld->learningEnabled); - } - - if(key == 'a') - { - tld->alternating = !tld->alternating; - printf("alternating: %d\n", tld->alternating); - } - - if(key == 'e') - { - tld->writeToFile(modelExportFile); - } - - if(key == 'i') - { - tld->readFromFile(modelPath); - } - - if(key == 'r') - { - CvRect box; - - if(getBBFromUser(img, box, gui) == PROGRAM_EXIT) - { - break; - } - - Rect r = Rect(box); - - tld->selectObject(grey, &r); - } - } - - if(saveDir != NULL) - { - char fileName[256]; - sprintf(fileName, "%s/%.5d.png", saveDir, imAcq->currentFrame - 1); - - cvSaveImage(fileName, img); - } - } - - if(reuseFrameOnce) - { - reuseFrameOnce = false; - } - } - - cvReleaseImage(&img); - img = NULL; - - if(exportModelAfterRun) - { - tld->writeToFile(modelExportFile); - } - - if(resultsFile) - { - fclose(resultsFile); - } + CvFont font; + cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, .5, .5, 0, 1, 8); + cvRectangle(img, cvPoint(0, 0), cvPoint(img->width, 50), black, CV_FILLED, 8, 0); + cvPutText(img, string, cvPoint(25, 25), &font, white); + + if (showForeground) + { + + for (size_t i = 0; i < tld->detectorCascade->detectionResult->fgList->size(); i++) + { + Rect r = tld->detectorCascade->detectionResult->fgList->at(i); + cvRectangle(img, r.tl(), r.br(), white, 1); + } + + } + + + if (showOutput) + { + gui->showImage(img); + char key = gui->getKey(); + + if (key == 'q') break; + + if (key == 'b') + { + + ForegroundDetector *fg = tld->detectorCascade->foregroundDetector; + + if (fg->bgImg.empty()) + { + fg->bgImg = grey.clone(); + } + else + { + fg->bgImg.release(); + } + } + + if (key == 'c') + { + //clear everything + tld->release(); + } + + if (key == 'l') + { + tld->learningEnabled = !tld->learningEnabled; + printf("LearningEnabled: %d\n", tld->learningEnabled); + } + + if (key == 'a') + { + tld->alternating = !tld->alternating; + printf("alternating: %d\n", tld->alternating); + } + + if (key == 'e') + { + tld->writeToFile(modelExportFile); + } + + if (key == 'i') + { + tld->readFromFile(modelPath); + } + + if (key == 'r') + { + CvRect box; + + if (getBBFromUser(img, box, gui) == PROGRAM_EXIT) + { + break; + } + + Rect r = Rect(box); + + tld->selectObject(grey, &r); + } + } + + if (saveDir != NULL) + { + char fileName[256]; + sprintf(fileName, "%s/%.5d.png", saveDir, imAcq->currentFrame - 1); + + cvSaveImage(fileName, img); + } + } + + if (reuseFrameOnce) + { + reuseFrameOnce = false; + } + } + + cvReleaseImage(&img); + img = NULL; + + if (exportModelAfterRun) + { + tld->writeToFile(modelExportFile); + } + + if (resultsFile) + { + fclose(resultsFile); + } } diff --git a/src/opentld/main/Main.h b/src/opentld/main/Main.h index 3c73ee9c..d660b0f2 100644 --- a/src/opentld/main/Main.h +++ b/src/opentld/main/Main.h @@ -72,7 +72,7 @@ class Main showTrajectory = false; trajectoryLength = 0; - selectManually = 0; + selectManually = 1; initialBB = NULL; showNotConfident = true;