java - Extract words in rectangles from text -

i struggling extract fast , efficiently words in rectangles bufferedimage.
example have following page : ( edit! ) image scanned, can contain noise, skewing , distortion.

how can extract following images without rectangle : ( edit! ) can use opencv or other library, i'm absolutely new advanced image processing techniques.

edit

i've used method suggested karlphillip here , works decent.
here code :

    package ro.ubbcluj.detection;  import java.awt.flowlayout; import java.awt.image.bufferedimage; import java.io.bytearrayinputstream; import java.io.ioexception; import java.io.inputstream; import java.util.arraylist; import java.util.list;  import javax.imageio.imageio; import javax.swing.imageicon; import javax.swing.jframe; import javax.swing.jlabel; import javax.swing.windowconstants;  import org.opencv.core.core; import org.opencv.core.mat; import org.opencv.core.matofbyte; import org.opencv.core.matofpoint; import org.opencv.core.point; import org.opencv.core.scalar; import org.opencv.core.size; import org.opencv.highgui.highgui; import org.opencv.imgproc.imgproc;  public class rectangledetection {  public static void main(string[] args) throws ioexception {     system.loadlibrary(core.native_library_name);     mat image = loadimage();     mat grayscale = converttograyscale(image);      mat treshold = tresholdimage(grayscale);     list<matofpoint> contours = findcontours(treshold);     mat contoursimage = fillcountours(contours, grayscale);     mat grayscalewithcontours = converttograyscale(contoursimage);     mat tresholdgrayscalewithcontours = tresholdimage(grayscalewithcontours);     mat eroded = erodeanddilate(tresholdgrayscalewithcontours);     list<matofpoint> squaresfound = findsquares(eroded);     mat squaresdrawn = rectangle.drawsquares(grayscale, squaresfound);     bufferedimage convertedimage = convertmattobufferedimage(squaresdrawn);     displayimage(convertedimage); }  private static list<matofpoint> findsquares(mat eroded) {     return rectangle.findsquares(eroded); }  private static mat erodeanddilate(mat input) {     int erosion_type = imgproc.morph_rect;     int erosion_size = 5;     mat result = new mat();     mat element = imgproc.getstructuringelement(erosion_type, new size(2 * erosion_size + 1, 2 * erosion_size + 1));     imgproc.erode(input, result, element);     imgproc.dilate(result, result, element);     return result; }  private static mat converttograyscale(mat input) {     mat grayscale = new mat();     imgproc.cvtcolor(input, grayscale, imgproc.color_bgr2gray);     return grayscale; }  private static mat fillcountours(list<matofpoint> contours, mat image) {     mat result = image.clone();     imgproc.cvtcolor(result, result, imgproc.color_gray2rgb);     (int = 0; < contours.size(); i++) {         imgproc.drawcontours(result, contours, i, new scalar(255, 0, 0), -1, 8, new mat(), 0, new point());     }     return result; }  private static list<matofpoint> findcontours(mat image) {     list<matofpoint> contours = new arraylist<>();     mat hierarchy = new mat();     imgproc.findcontours(image, contours, hierarchy, imgproc.retr_tree, imgproc.chain_approx_none);     return contours; }  private static mat detectlineshough(mat img) {     mat lines = new mat();     int threshold = 80;     int minlinelength = 10;     int maxlinegap = 5;     double rho = 0.4;     imgproc.houghlinesp(img, lines, rho, math.pi / 180, threshold, minlinelength, maxlinegap);     imgproc.cvtcolor(img, img, imgproc.color_gray2rgb);     system.out.println(lines.cols());     (int x = 0; x < lines.cols(); x++) {         double[] vec = lines.get(0, x);         double x1 = vec[0], y1 = vec[1], x2 = vec[2], y2 = vec[3];         point start = new point(x1, y1);         point end = new point(x2, y2);         core.line(lines, start, end, new scalar(0, 255, 0), 3);     }     return img; }  static bufferedimage convertmattobufferedimage(mat mat) throws ioexception {     matofbyte matofbyte = new matofbyte();     highgui.imencode(".jpg", mat, matofbyte);     byte[] bytearray = matofbyte.toarray();     inputstream in = new bytearrayinputstream(bytearray);     return imageio.read(in);  }  static void displayimage(bufferedimage image) {     jframe frame = new jframe();     frame.getcontentpane().setlayout(new flowlayout());     frame.getcontentpane().add(new jlabel(new imageicon(image)));     frame.setdefaultcloseoperation(windowconstants.exit_on_close);     frame.pack();     frame.setvisible(true);  }  private static mat tresholdimage(mat img) {     mat treshold = new mat();     imgproc.threshold(img, treshold, 225, 255, imgproc.thresh_binary_inv);     return treshold; }  private static mat tresholdimage2(mat img) {     mat treshold = new mat();     imgproc.threshold(img, treshold, -1, 255, imgproc.thresh_binary_inv + imgproc.thresh_otsu);     return treshold; }  private static mat loadimage() {     return highgui             .imread("e:\\programs\\eclipse workspace\\licentaworkspace\\opencvrectangledetection\\src\\img\\form3.jpg"); } 

}

, rectangle class

    package ro.ubbcluj.detection;  import java.awt.image.bufferedimage; import java.io.ioexception; import java.util.arraylist; import java.util.list;  import org.opencv.core.core; import org.opencv.core.mat; import org.opencv.core.matofpoint; import org.opencv.core.matofpoint2f; import org.opencv.core.point; import org.opencv.core.scalar; import org.opencv.core.size; import org.opencv.imgproc.imgproc;  public class rectangle { static list<matofpoint> findsquares(mat input) {     mat pyr = new mat();     mat timg = new mat();      // down-scale , up-scale image filter out small noises     imgproc.pyrdown(input, pyr, new size(input.cols() / 2, input.rows() / 2));     imgproc.pyrup(pyr, timg, input.size());     // apply canny threshold of 50     imgproc.canny(timg, timg, 0, 50, 5, true);      // dilate canny output remove potential holes between edge segments     imgproc.dilate(timg, timg, new mat(), new point(-1, -1), 1);      // find contours , store them list     mat hierarchy = new mat();     list<matofpoint> contours = new arraylist<>();     imgproc.findcontours(timg, contours, hierarchy, imgproc.retr_list, imgproc.chain_approx_simple);     list<matofpoint> squaresresult = new arraylist<matofpoint>();     (int = 0; < contours.size(); i++) {          // approximate contour accuracy proportional contour         // perimeter         matofpoint2f contour = new matofpoint2f(contours.get(i).toarray());         matofpoint2f approx = new matofpoint2f();         double epsilon = imgproc.arclength(contour, true) * 0.02;         boolean closed = true;         imgproc.approxpolydp(contour, approx, epsilon, closed);         list<point> approxcurvelist = approx.tolist();          // square contours should have 4 vertices after approximation         // relatively large area (to filter out noisy contours)         // , convex.         // note: absolute value of area used because         // area may positive or negative - in accordance         // contour orientation         boolean aproxsize = approx.rows() == 4;         boolean largearea = math.abs(imgproc.contourarea(approx)) > 200;         boolean isconvex = imgproc.iscontourconvex(new matofpoint(approx.toarray()));         if (aproxsize && largearea && isconvex) {             double maxcosine = 0;             (int j = 2; j < 5; j++) {                 // find maximum cosine of angle between joint edges                 double cosine = math.abs(getangle(approxcurvelist.get(j % 4), approxcurvelist.get(j - 2),                         approxcurvelist.get(j - 1)));                 maxcosine = math.max(maxcosine, cosine);             }             // if cosines of angles small             // (all angles ~90 degree) write quandrange             // vertices resultant sequence             if (maxcosine < 0.3) {                 point[] points = approx.toarray();                 squaresresult.add(new matofpoint(points));             }         }     }     return squaresresult; }  // angle: helper function. // finds cosine of angle between vectors pt0->pt1 , pt0->pt2. private static double getangle(point point1, point point2, point point0) {     double dx1 = point1.x - point0.x;     double dy1 = point1.y - point0.y;     double dx2 = point2.x - point0.x;     double dy2 = point2.y - point0.y;     return (dx1 * dx2 + dy1 * dy2) / math.sqrt((dx1 * dx1 + dy1 * dy1) * (dx2 * dx2 + dy2 * dy2) + 1e-10); }  public static mat drawsquares(mat image, list<matofpoint> squares) {     mat result = new mat();     imgproc.cvtcolor(image, result, imgproc.color_gray2rgb);     int thickness = 2;     core.polylines(result, squares, false, new scalar(0, 255, 0), thickness);     return result; } } 

example of result :

... though, doesn't work great smaller images :

maybe enhancements can suggested? or how make algorithm faster in case have batch of images process?

i did following program in c++ using opencv (i'm not familiar java+opencv). i've included output 2 sample images have provided. may have adjust thresholds in contour filtering section other images.

#include "stdafx.h"  #include <opencv2/core/core.hpp> #include <opencv2/highgui/highgui.hpp> #include <opencv2/imgproc/imgproc.hpp> #include <iostream>  using namespace cv; using namespace std;  int _tmain(int argc, _tchar* argv[]) {     // load image grayscale     mat im = imread(input_file, cv_load_image_grayscale);      mat morph;     // morphological closing column filter : retain large vertical edges     mat morphkernelv = getstructuringelement(morph_rect, size(1, 7));     morphologyex(im, morph, morph_close, morphkernelv);      mat bwv;     // binarize: contain large vertical edges     threshold(morph, bwv, 0, 255.0, cv_thresh_binary | cv_thresh_otsu);      // morphological closing row filter : retain large horizontal edges     mat morphkernelh = getstructuringelement(morph_rect, size(7, 1));     morphologyex(im, morph, morph_close, morphkernelh);      mat bwh;     // binarize: contain large horizontal edges     threshold(morph, bwh, 0, 255.0, cv_thresh_binary | cv_thresh_otsu);      // combine virtical , horizontal edges     mat bw = bwv & bwh;     threshold(bw, bw, 128.0, 255.0, cv_thresh_binary_inv);      // illustration     mat rgb;     cvtcolor(im, rgb, cv_gray2bgr);      // find contours     vector<vector<point>> contours;     vector<vec4i> hierarchy;     findcontours(bw, contours, hierarchy, cv_retr_ccomp, cv_chain_approx_simple, point(0, 0));     // filter contours area obtain boxes     double areathl = bw.rows * .04 * bw.cols * .06;     double areathh = bw.rows * .7 * bw.cols * .7;     double area = 0;     for(int idx = 0; idx >= 0; idx = hierarchy[idx][0])     {         area = contourarea(contours[idx]);          if (area > areathl && area < areathh)         {             drawcontours(rgb, contours, idx, scalar(0, 0, 255), 2, 8, hierarchy);             // take bounding rectangle. better use filled countour mask             // extract rectangle because won't stray elements             rect rect = boundingrect(contours[idx]);             cout << "rect: (" << rect.x << ", " << rect.y << ") " << rect.width << " x " << rect.height << endl;             mat imrect(im, rect);         }     }      return 0; } 

result first image:

result second image: