java - Extract words in rectangles from text -
i struggling extract fast , efficiently words in rectangles bufferedimage.
example have following page : ( edit! ) image scanned, can contain noise, skewing , distortion.
how can extract following images without rectangle : ( edit! ) can use opencv or other library, i'm absolutely new advanced image processing techniques. 
edit
i've used method suggested karlphillip here , works decent.
here code :
package ro.ubbcluj.detection; import java.awt.flowlayout; import java.awt.image.bufferedimage; import java.io.bytearrayinputstream; import java.io.ioexception; import java.io.inputstream; import java.util.arraylist; import java.util.list; import javax.imageio.imageio; import javax.swing.imageicon; import javax.swing.jframe; import javax.swing.jlabel; import javax.swing.windowconstants; import org.opencv.core.core; import org.opencv.core.mat; import org.opencv.core.matofbyte; import org.opencv.core.matofpoint; import org.opencv.core.point; import org.opencv.core.scalar; import org.opencv.core.size; import org.opencv.highgui.highgui; import org.opencv.imgproc.imgproc; public class rectangledetection { public static void main(string[] args) throws ioexception { system.loadlibrary(core.native_library_name); mat image = loadimage(); mat grayscale = converttograyscale(image); mat treshold = tresholdimage(grayscale); list<matofpoint> contours = findcontours(treshold); mat contoursimage = fillcountours(contours, grayscale); mat grayscalewithcontours = converttograyscale(contoursimage); mat tresholdgrayscalewithcontours = tresholdimage(grayscalewithcontours); mat eroded = erodeanddilate(tresholdgrayscalewithcontours); list<matofpoint> squaresfound = findsquares(eroded); mat squaresdrawn = rectangle.drawsquares(grayscale, squaresfound); bufferedimage convertedimage = convertmattobufferedimage(squaresdrawn); displayimage(convertedimage); } private static list<matofpoint> findsquares(mat eroded) { return rectangle.findsquares(eroded); } private static mat erodeanddilate(mat input) { int erosion_type = imgproc.morph_rect; int erosion_size = 5; mat result = new mat(); mat element = imgproc.getstructuringelement(erosion_type, new size(2 * erosion_size + 1, 2 * erosion_size + 1)); imgproc.erode(input, result, element); imgproc.dilate(result, result, element); return result; } private static mat converttograyscale(mat input) { mat grayscale = new mat(); imgproc.cvtcolor(input, grayscale, imgproc.color_bgr2gray); return grayscale; } private static mat fillcountours(list<matofpoint> contours, mat image) { mat result = image.clone(); imgproc.cvtcolor(result, result, imgproc.color_gray2rgb); (int = 0; < contours.size(); i++) { imgproc.drawcontours(result, contours, i, new scalar(255, 0, 0), -1, 8, new mat(), 0, new point()); } return result; } private static list<matofpoint> findcontours(mat image) { list<matofpoint> contours = new arraylist<>(); mat hierarchy = new mat(); imgproc.findcontours(image, contours, hierarchy, imgproc.retr_tree, imgproc.chain_approx_none); return contours; } private static mat detectlineshough(mat img) { mat lines = new mat(); int threshold = 80; int minlinelength = 10; int maxlinegap = 5; double rho = 0.4; imgproc.houghlinesp(img, lines, rho, math.pi / 180, threshold, minlinelength, maxlinegap); imgproc.cvtcolor(img, img, imgproc.color_gray2rgb); system.out.println(lines.cols()); (int x = 0; x < lines.cols(); x++) { double[] vec = lines.get(0, x); double x1 = vec[0], y1 = vec[1], x2 = vec[2], y2 = vec[3]; point start = new point(x1, y1); point end = new point(x2, y2); core.line(lines, start, end, new scalar(0, 255, 0), 3); } return img; } static bufferedimage convertmattobufferedimage(mat mat) throws ioexception { matofbyte matofbyte = new matofbyte(); highgui.imencode(".jpg", mat, matofbyte); byte[] bytearray = matofbyte.toarray(); inputstream in = new bytearrayinputstream(bytearray); return imageio.read(in); } static void displayimage(bufferedimage image) { jframe frame = new jframe(); frame.getcontentpane().setlayout(new flowlayout()); frame.getcontentpane().add(new jlabel(new imageicon(image))); frame.setdefaultcloseoperation(windowconstants.exit_on_close); frame.pack(); frame.setvisible(true); } private static mat tresholdimage(mat img) { mat treshold = new mat(); imgproc.threshold(img, treshold, 225, 255, imgproc.thresh_binary_inv); return treshold; } private static mat tresholdimage2(mat img) { mat treshold = new mat(); imgproc.threshold(img, treshold, -1, 255, imgproc.thresh_binary_inv + imgproc.thresh_otsu); return treshold; } private static mat loadimage() { return highgui .imread("e:\\programs\\eclipse workspace\\licentaworkspace\\opencvrectangledetection\\src\\img\\form3.jpg"); } }
, rectangle class
package ro.ubbcluj.detection; import java.awt.image.bufferedimage; import java.io.ioexception; import java.util.arraylist; import java.util.list; import org.opencv.core.core; import org.opencv.core.mat; import org.opencv.core.matofpoint; import org.opencv.core.matofpoint2f; import org.opencv.core.point; import org.opencv.core.scalar; import org.opencv.core.size; import org.opencv.imgproc.imgproc; public class rectangle { static list<matofpoint> findsquares(mat input) { mat pyr = new mat(); mat timg = new mat(); // down-scale , up-scale image filter out small noises imgproc.pyrdown(input, pyr, new size(input.cols() / 2, input.rows() / 2)); imgproc.pyrup(pyr, timg, input.size()); // apply canny threshold of 50 imgproc.canny(timg, timg, 0, 50, 5, true); // dilate canny output remove potential holes between edge segments imgproc.dilate(timg, timg, new mat(), new point(-1, -1), 1); // find contours , store them list mat hierarchy = new mat(); list<matofpoint> contours = new arraylist<>(); imgproc.findcontours(timg, contours, hierarchy, imgproc.retr_list, imgproc.chain_approx_simple); list<matofpoint> squaresresult = new arraylist<matofpoint>(); (int = 0; < contours.size(); i++) { // approximate contour accuracy proportional contour // perimeter matofpoint2f contour = new matofpoint2f(contours.get(i).toarray()); matofpoint2f approx = new matofpoint2f(); double epsilon = imgproc.arclength(contour, true) * 0.02; boolean closed = true; imgproc.approxpolydp(contour, approx, epsilon, closed); list<point> approxcurvelist = approx.tolist(); // square contours should have 4 vertices after approximation // relatively large area (to filter out noisy contours) // , convex. // note: absolute value of area used because // area may positive or negative - in accordance // contour orientation boolean aproxsize = approx.rows() == 4; boolean largearea = math.abs(imgproc.contourarea(approx)) > 200; boolean isconvex = imgproc.iscontourconvex(new matofpoint(approx.toarray())); if (aproxsize && largearea && isconvex) { double maxcosine = 0; (int j = 2; j < 5; j++) { // find maximum cosine of angle between joint edges double cosine = math.abs(getangle(approxcurvelist.get(j % 4), approxcurvelist.get(j - 2), approxcurvelist.get(j - 1))); maxcosine = math.max(maxcosine, cosine); } // if cosines of angles small // (all angles ~90 degree) write quandrange // vertices resultant sequence if (maxcosine < 0.3) { point[] points = approx.toarray(); squaresresult.add(new matofpoint(points)); } } } return squaresresult; } // angle: helper function. // finds cosine of angle between vectors pt0->pt1 , pt0->pt2. private static double getangle(point point1, point point2, point point0) { double dx1 = point1.x - point0.x; double dy1 = point1.y - point0.y; double dx2 = point2.x - point0.x; double dy2 = point2.y - point0.y; return (dx1 * dx2 + dy1 * dy2) / math.sqrt((dx1 * dx1 + dy1 * dy1) * (dx2 * dx2 + dy2 * dy2) + 1e-10); } public static mat drawsquares(mat image, list<matofpoint> squares) { mat result = new mat(); imgproc.cvtcolor(image, result, imgproc.color_gray2rgb); int thickness = 2; core.polylines(result, squares, false, new scalar(0, 255, 0), thickness); return result; } } example of result :
... though, doesn't work great smaller images : 
maybe enhancements can suggested? or how make algorithm faster in case have batch of images process?
i did following program in c++ using opencv (i'm not familiar java+opencv). i've included output 2 sample images have provided. may have adjust thresholds in contour filtering section other images.
#include "stdafx.h" #include <opencv2/core/core.hpp> #include <opencv2/highgui/highgui.hpp> #include <opencv2/imgproc/imgproc.hpp> #include <iostream> using namespace cv; using namespace std; int _tmain(int argc, _tchar* argv[]) { // load image grayscale mat im = imread(input_file, cv_load_image_grayscale); mat morph; // morphological closing column filter : retain large vertical edges mat morphkernelv = getstructuringelement(morph_rect, size(1, 7)); morphologyex(im, morph, morph_close, morphkernelv); mat bwv; // binarize: contain large vertical edges threshold(morph, bwv, 0, 255.0, cv_thresh_binary | cv_thresh_otsu); // morphological closing row filter : retain large horizontal edges mat morphkernelh = getstructuringelement(morph_rect, size(7, 1)); morphologyex(im, morph, morph_close, morphkernelh); mat bwh; // binarize: contain large horizontal edges threshold(morph, bwh, 0, 255.0, cv_thresh_binary | cv_thresh_otsu); // combine virtical , horizontal edges mat bw = bwv & bwh; threshold(bw, bw, 128.0, 255.0, cv_thresh_binary_inv); // illustration mat rgb; cvtcolor(im, rgb, cv_gray2bgr); // find contours vector<vector<point>> contours; vector<vec4i> hierarchy; findcontours(bw, contours, hierarchy, cv_retr_ccomp, cv_chain_approx_simple, point(0, 0)); // filter contours area obtain boxes double areathl = bw.rows * .04 * bw.cols * .06; double areathh = bw.rows * .7 * bw.cols * .7; double area = 0; for(int idx = 0; idx >= 0; idx = hierarchy[idx][0]) { area = contourarea(contours[idx]); if (area > areathl && area < areathh) { drawcontours(rgb, contours, idx, scalar(0, 0, 255), 2, 8, hierarchy); // take bounding rectangle. better use filled countour mask // extract rectangle because won't stray elements rect rect = boundingrect(contours[idx]); cout << "rect: (" << rect.x << ", " << rect.y << ") " << rect.width << " x " << rect.height << endl; mat imrect(im, rect); } } return 0; } result first image:
result second image:
Comments
Post a Comment