diff --git a/ACP/ACP/AdvancedProcessor.cpp b/ACP/ACP/AdvancedProcessor.cpp index 335fb35..fed5223 100644 --- a/ACP/ACP/AdvancedProcessor.cpp +++ b/ACP/ACP/AdvancedProcessor.cpp @@ -10,198 +10,202 @@ // Simple compute kernel which computes the square of an input array // -const char *KernelSource = "\n" \ -"__kernel void extractText ( \n"\ -" __global Mat input, \n"\ -" __global string output, \n"\ -" int count) \n"\ -" { \n"\ -" tessAPI = new tesseract::TessBaseAPI(); \n"\ -" tessAPI->SetImage(input.data, input.cols, input.rows, \n"\ -" input.channels(), input.step); \n"\ -" int i = get_global_id(0); \n"\ -" if(i < count) { \n"\ -" output[i] = tessAPI->GetUTF8Text(); \n"\ -" tessAPI->End(); \n"\ -" } \n"\ -" } \n"\ -"\n"; +//const char *KernelSource = "\n" \ +//"__kernel void extractText ( \n"\ +//" __global Mat input, \n"\ +//" __global string output, \n"\ +//" int count) \n"\ +//" { \n"\ +//" tessAPI = new tesseract::TessBaseAPI(); \n"\ +//" tessAPI->SetImage(input.data, input.cols, input.rows, \n"\ +//" input.channels(), input.step); \n"\ +//" int i = get_global_id(0); \n"\ +//" if(i < count) { \n"\ +//" output[i] = tessAPI->GetUTF8Text(); \n"\ +//" tessAPI->End(); \n"\ +//" } \n"\ +//" } \n"\ +//"\n"; //////////////////////////////////////////////////////////////////////////////// + + +// This class was created to use OpenCL for extracting text in parallel. This method was discarded because +// Tesseract is not written in C++. The code is not deleted just in case some workaround can be devised. int AdvancedProcessor :: processAcrossCores() { - int err; // error code returned from api calls - - float data[DATA_SIZE]; // original data set given to device - float results[DATA_SIZE]; // results returned from device - unsigned int correct; // number of correct results returned - - size_t global; // global domain size for our calculation - size_t local; // local domain size for our calculation - - // Setup objects - cl_device_id device_id; // compute device id - cl_context context; // compute context - cl_command_queue commands; // compute command queue - // Execution objects - cl_program program; // compute program - cl_kernel kernel; // compute kernel - - cl_mem input; // device memory used for the input array - cl_mem output; // device memory used for the output array - - // Fill our data set with random float values - // - int i = 0; - unsigned int count = DATA_SIZE; - for(i = 0; i < count; i++) - data[i] = rand() / (float)RAND_MAX; - - // Connect to a compute device - // Give me a GPU or a CPU device - int gpu = 1; - err = clGetDeviceIDs(NULL, gpu ? CL_DEVICE_TYPE_GPU : CL_DEVICE_TYPE_CPU, 1, &device_id, NULL); - if (err != CL_SUCCESS) - { - printf("Error: Failed to create a device group!\n"); - return EXIT_FAILURE; - } - - // Create a compute context - // - context = clCreateContext(0, 1, &device_id, NULL, NULL, &err); - if (!context) - { - printf("Error: Failed to create a compute context!\n"); - return EXIT_FAILURE; - } - - // Create a command commands - // One command queue for each device - commands = clCreateCommandQueue(context, device_id, 0, &err); - if (!commands) - { - printf("Error: Failed to create a command commands!\n"); - return EXIT_FAILURE; - } - - // Create the compute program from the source buffer - // - program = clCreateProgramWithSource(context, 1, (const char **) & KernelSource, NULL, &err); - if (!program) - { - printf("Error: Failed to create compute program!\n"); - return EXIT_FAILURE; - } - - // Build the program executable - // - err = clBuildProgram(program, 0, NULL, NULL, NULL, NULL); - if (err != CL_SUCCESS) - { - size_t len; - char buffer[2048]; - - printf("Error: Failed to build program executable!\n"); - clGetProgramBuildInfo(program, device_id, CL_PROGRAM_BUILD_LOG, sizeof(buffer), buffer, &len); - printf("%s\n", buffer); - exit(1); - } - - // Create the compute kernel in the program we wish to run - // - kernel = clCreateKernel(program, "square", &err); - if (!kernel || err != CL_SUCCESS) - { - printf("Error: Failed to create compute kernel!\n"); - exit(1); - } - - // Create the input and output arrays in device memory for our calculation - // size in bytes - input = clCreateBuffer(context, CL_MEM_READ_ONLY, sizeof(float) * count, NULL, NULL); - output = clCreateBuffer(context, CL_MEM_WRITE_ONLY, sizeof(float) * count, NULL, NULL); - if (!input || !output) - { - printf("Error: Failed to allocate device memory!\n"); - exit(1); - } - - // Write our data set into the input array in device memory - // Refer evernote for syntax - err = clEnqueueWriteBuffer(commands, input, CL_TRUE, 0, sizeof(float) * count, data, 0, NULL, NULL); - if (err != CL_SUCCESS) - { - printf("Error: Failed to write to source array!\n"); - exit(1); - } - - // Set the arguments to our compute kernel - // - err = 0; - err = clSetKernelArg(kernel, 0, sizeof(cl_mem), &input); - err |= clSetKernelArg(kernel, 1, sizeof(cl_mem), &output); - err |= clSetKernelArg(kernel, 2, sizeof(unsigned int), &count); - if (err != CL_SUCCESS) - { - printf("Error: Failed to set kernel arguments! %d\n", err); - exit(1); - } - - // Get the maximum work group size for executing the kernel on the device - // - err = clGetKernelWorkGroupInfo(kernel, device_id, CL_KERNEL_WORK_GROUP_SIZE, sizeof(local), &local, NULL); - if (err != CL_SUCCESS) - { - printf("Error: Failed to retrieve kernel work group info! %d\n", err); - exit(1); - } - - // Execute the kernel over the entire range of our 1d input data set - // using the maximum number of work group items for this device - // Enqueue the work - global = count; - err = clEnqueueNDRangeKernel(commands, kernel, 1, NULL, &global, &local, 0, NULL, NULL); - if (err) - { - printf("Error: Failed to execute kernel!\n"); - return EXIT_FAILURE; - } - - // Wait for the command commands to get serviced before reading back results - // - clFinish(commands); - - // Read back the results from the device to verify the output - // - err = clEnqueueReadBuffer( commands, output, CL_TRUE, 0, sizeof(float) * count, results, 0, NULL, NULL ); - if (err != CL_SUCCESS) - { - printf("Error: Failed to read output array! %d\n", err); - exit(1); - } - - // Validate our results - // - correct = 0; - for(i = 0; i < count; i++) - { - if(results[i] == data[i] * data[i]) - correct++; - } - - // Print a brief summary detailing the results - // - printf("Computed '%d/%d' correct values!\n", correct, count); - - // Shutdown and cleanup - // - clReleaseMemObject(input); - clReleaseMemObject(output); - clReleaseProgram(program); - clReleaseKernel(kernel); - clReleaseCommandQueue(commands); - clReleaseContext(context); - +// int err; // error code returned from api calls +// +// float data[DATA_SIZE]; // original data set given to device +// float results[DATA_SIZE]; // results returned from device +// unsigned int correct; // number of correct results returned +// +// size_t global; // global domain size for our calculation +// size_t local; // local domain size for our calculation +// +// // Setup objects +// cl_device_id device_id; // compute device id +// cl_context context; // compute context +// cl_command_queue commands; // compute command queue +// // Execution objects +// cl_program program; // compute program +// cl_kernel kernel; // compute kernel +// +// cl_mem input; // device memory used for the input array +// cl_mem output; // device memory used for the output array +// +// // Fill our data set with random float values +// // +// int i = 0; +// unsigned int count = DATA_SIZE; +// for(i = 0; i < count; i++) +// data[i] = rand() / (float)RAND_MAX; +// +// // Connect to a compute device +// // Give me a GPU or a CPU device +// int gpu = 1; +// err = clGetDeviceIDs(NULL, gpu ? CL_DEVICE_TYPE_GPU : CL_DEVICE_TYPE_CPU, 1, &device_id, NULL); +// if (err != CL_SUCCESS) +// { +// printf("Error: Failed to create a device group!\n"); +// return EXIT_FAILURE; +// } +// +// // Create a compute context +// // +// context = clCreateContext(0, 1, &device_id, NULL, NULL, &err); +// if (!context) +// { +// printf("Error: Failed to create a compute context!\n"); +// return EXIT_FAILURE; +// } +// +// // Create a command commands +// // One command queue for each device +// commands = clCreateCommandQueue(context, device_id, 0, &err); +// if (!commands) +// { +// printf("Error: Failed to create a command commands!\n"); +// return EXIT_FAILURE; +// } +// +// // Create the compute program from the source buffer +// // +// program = clCreateProgramWithSource(context, 1, (const char **) & KernelSource, NULL, &err); +// if (!program) +// { +// printf("Error: Failed to create compute program!\n"); +// return EXIT_FAILURE; +// } +// +// // Build the program executable +// // +// err = clBuildProgram(program, 0, NULL, NULL, NULL, NULL); +// if (err != CL_SUCCESS) +// { +// size_t len; +// char buffer[2048]; +// +// printf("Error: Failed to build program executable!\n"); +// clGetProgramBuildInfo(program, device_id, CL_PROGRAM_BUILD_LOG, sizeof(buffer), buffer, &len); +// printf("%s\n", buffer); +// exit(1); +// } +// +// // Create the compute kernel in the program we wish to run +// // +// kernel = clCreateKernel(program, "square", &err); +// if (!kernel || err != CL_SUCCESS) +// { +// printf("Error: Failed to create compute kernel!\n"); +// exit(1); +// } +// +// // Create the input and output arrays in device memory for our calculation +// // size in bytes +// input = clCreateBuffer(context, CL_MEM_READ_ONLY, sizeof(float) * count, NULL, NULL); +// output = clCreateBuffer(context, CL_MEM_WRITE_ONLY, sizeof(float) * count, NULL, NULL); +// if (!input || !output) +// { +// printf("Error: Failed to allocate device memory!\n"); +// exit(1); +// } +// +// // Write our data set into the input array in device memory +// // Refer evernote for syntax +// err = clEnqueueWriteBuffer(commands, input, CL_TRUE, 0, sizeof(float) * count, data, 0, NULL, NULL); +// if (err != CL_SUCCESS) +// { +// printf("Error: Failed to write to source array!\n"); +// exit(1); +// } +// +// // Set the arguments to our compute kernel +// // +// err = 0; +// err = clSetKernelArg(kernel, 0, sizeof(cl_mem), &input); +// err |= clSetKernelArg(kernel, 1, sizeof(cl_mem), &output); +// err |= clSetKernelArg(kernel, 2, sizeof(unsigned int), &count); +// if (err != CL_SUCCESS) +// { +// printf("Error: Failed to set kernel arguments! %d\n", err); +// exit(1); +// } +// +// // Get the maximum work group size for executing the kernel on the device +// // +// err = clGetKernelWorkGroupInfo(kernel, device_id, CL_KERNEL_WORK_GROUP_SIZE, sizeof(local), &local, NULL); +// if (err != CL_SUCCESS) +// { +// printf("Error: Failed to retrieve kernel work group info! %d\n", err); +// exit(1); +// } +// +// // Execute the kernel over the entire range of our 1d input data set +// // using the maximum number of work group items for this device +// // Enqueue the work +// global = count; +// err = clEnqueueNDRangeKernel(commands, kernel, 1, NULL, &global, &local, 0, NULL, NULL); +// if (err) +// { +// printf("Error: Failed to execute kernel!\n"); +// return EXIT_FAILURE; +// } +// +// // Wait for the command commands to get serviced before reading back results +// // +// clFinish(commands); +// +// // Read back the results from the device to verify the output +// // +// err = clEnqueueReadBuffer( commands, output, CL_TRUE, 0, sizeof(float) * count, results, 0, NULL, NULL ); +// if (err != CL_SUCCESS) +// { +// printf("Error: Failed to read output array! %d\n", err); +// exit(1); +// } +// +// // Validate our results +// // +// correct = 0; +// for(i = 0; i < count; i++) +// { +// if(results[i] == data[i] * data[i]) +// correct++; +// } +// +// // Print a brief summary detailing the results +// // +// printf("Computed '%d/%d' correct values!\n", correct, count); +// +// // Shutdown and cleanup +// // +// clReleaseMemObject(input); +// clReleaseMemObject(output); +// clReleaseProgram(program); +// clReleaseKernel(kernel); +// clReleaseCommandQueue(commands); +// clReleaseContext(context); +// return 0; } diff --git a/ACP/ACP/Processor.cpp b/ACP/ACP/Processor.cpp index a3cc76d..88b52e4 100644 --- a/ACP/ACP/Processor.cpp +++ b/ACP/ACP/Processor.cpp @@ -37,10 +37,12 @@ void Processor::resizeImage(int width, int height) { void Processor::prepareImageForOCR() { cv::GaussianBlur(page.getImage(), page.getImage(), Size(3, 3), 0, 0); + // Otsu's threshold already applied by Tesseract - use with caution cv::adaptiveThreshold(page.getImage(), page.getImage(), 255.f, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 5, 4.1f); } +// For camera setup as given in the demo video of ACP-Paper void Processor::rotateImageClockwise(double angle) { Size srcSize = page.getImage().size(); Size dstSize(srcSize.height, srcSize.width); @@ -70,6 +72,7 @@ string Processor::extractTextFromImage(Mat image) { } +// TODO: requires testing string Processor::replaceUnwantedCharactersWithSpace(string text) { string allowedCharacters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-+_,.:;[]{}<>/\'\"? "; size_t nonalpha = text.find_first_not_of(allowedCharacters); @@ -82,6 +85,7 @@ string Processor::replaceUnwantedCharactersWithSpace(string text) { } +// For removing blemishes Mat Processor::erodeImage(int, void*) { int erosionType = MORPH_RECT; int erosionSize = 13; @@ -104,6 +108,7 @@ Mat Processor::dilateImage(int, void*) { } +// Splits parent image into several sub-parts TODO: fails sometimes, requires testing vector Processor::cutImageGivenWhiteLineLocations(vector whiteLineLocations) { vector images; Mat image = this->getPage().getImage(); @@ -119,6 +124,7 @@ vector Processor::cutImageGivenWhiteLineLocations(vector whiteLineLoca } +// Calculates the sum of the pixels of each line, compares against adjacent lines and returns only those that are actually white vector Processor::findWhiteLines(Mat img) { vector whiteLines; map actualWhiteLines; @@ -146,6 +152,8 @@ vector Processor::findWhiteLines(Mat img) { } +// Returns the splitting locations (average of first white line after and last white line +// before line containing black pixels) from a list of white lines identified vector Processor::getSplittingLocations() { vector splittingLocations; Mat image = this->getPage().getImage(); @@ -174,7 +182,7 @@ vector Processor::getSplittingLocations() { -// Unused +// Currently unused methods vector Processor::cutImage(int x_coord, int y_coord) { Mat bigImage = page.getImage(); Mat smallImage = Mat(bigImage, Rect(0, 0, x_coord, y_coord)); @@ -192,6 +200,7 @@ vector Processor::cutImage(int x_coord, int y_coord) { } +// The following two methods were tried to identify white regions in the image vector Processor::detectLetters(cv::Mat img) { std::vector boundRect; cv::Mat img_gray, img_sobel, img_threshold, element; @@ -263,6 +272,7 @@ void Processor::displayImage(Mat image) { } +// Needs to be added for improving accuracy void Processor::deskewImage(Mat image, double angle, Mat &rotated) { bitwise_not(image, image); vector points; diff --git a/ACP/ACP/main.cpp b/ACP/ACP/main.cpp index 8cc624d..97a7546 100644 --- a/ACP/ACP/main.cpp +++ b/ACP/ACP/main.cpp @@ -31,7 +31,8 @@ void *extractText(void *threadArg) { int main(int argc, const char *argv[]) { Camera cam; - Mat image = cam.getImageFileInput("/Users/raghav/Desktop/11.png"); + // File name to be changed depending on the machine + Mat image = cam.getImageFileInput("/Users/raghav/Desktop/12.png"); // Initialise Page parameters Page page; @@ -41,22 +42,21 @@ int main(int argc, const char *argv[]) { Processor processor; processor.setPage(page); - vector whiteLineLocations = processor.getSplittingLocations(); - vector cutImages = processor.cutImageGivenWhiteLineLocations(whiteLineLocations); - -// for (int i = 0; i < cutImages.size(); i++) { -// cout << cutImages.size() << endl; -// imshow("Image", cutImages[i]); -// waitKey(); -// } + // 1. To test without speedup, comment out one of the two methods to obtain correct results // Send to OCR // string output = processor.extractTextFromImage(image); // cout << output << endl; + // end of 1. + + + // 2. To test with speedup + vector whiteLineLocations = processor.getSplittingLocations(); + vector cutImages = processor.cutImageGivenWhiteLineLocations(whiteLineLocations); int numberOfThreads = (int)cutImages.size(); - struct threadData td[22]; - pthread_t threads[22]; + struct threadData td[50]; + pthread_t threads[50]; for (int i = 1; i < numberOfThreads; i++) { td[i].id = i; @@ -64,11 +64,12 @@ int main(int argc, const char *argv[]) { td[i].image = cutImages[i]; int statusFail = pthread_create(&threads[i], NULL, extractText, (void*)&td[i]); if (statusFail) { - cout << "Error: Unable to create thread," << statusFail << endl; + cout << "Error: Unable to create thread." << statusFail << endl; exit(-1); } } pthread_exit(NULL); + // end of 2. return 0; }