193

I successfully implemented the OpenCV square-detection example in my test application, but now need to filter the output, because it's quite messy - or is my code wrong?

I'm interested in the four corner points of the paper for skew reduction (like that) and further processing …

Input & Output: Input & Output

Original image:

click

Code:

double angle( cv::Point pt1, cv::Point pt2, cv::Point pt0 ) {
    double dx1 = pt1.x - pt0.x;
    double dy1 = pt1.y - pt0.y;
    double dx2 = pt2.x - pt0.x;
    double dy2 = pt2.y - pt0.y;
    return (dx1*dx2 + dy1*dy2)/sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
}

- (std::vector<std::vector<cv::Point> >)findSquaresInImage:(cv::Mat)_image
{
    std::vector<std::vector<cv::Point> > squares;
    cv::Mat pyr, timg, gray0(_image.size(), CV_8U), gray;
    int thresh = 50, N = 11;
    cv::pyrDown(_image, pyr, cv::Size(_image.cols/2, _image.rows/2));
    cv::pyrUp(pyr, timg, _image.size());
    std::vector<std::vector<cv::Point> > contours;
    for( int c = 0; c < 3; c++ ) {
        int ch[] = {c, 0};
        mixChannels(&timg, 1, &gray0, 1, ch, 1);
        for( int l = 0; l < N; l++ ) {
            if( l == 0 ) {
                cv::Canny(gray0, gray, 0, thresh, 5);
                cv::dilate(gray, gray, cv::Mat(), cv::Point(-1,-1));
            }
            else {
                gray = gray0 >= (l+1)*255/N;
            }
            cv::findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);
            std::vector<cv::Point> approx;
            for( size_t i = 0; i < contours.size(); i++ )
            {
                cv::approxPolyDP(cv::Mat(contours[i]), approx, arcLength(cv::Mat(contours[i]), true)*0.02, true);
                if( approx.size() == 4 && fabs(contourArea(cv::Mat(approx))) > 1000 && cv::isContourConvex(cv::Mat(approx))) {
                    double maxCosine = 0;

                    for( int j = 2; j < 5; j++ )
                    {
                        double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
                        maxCosine = MAX(maxCosine, cosine);
                    }

                    if( maxCosine < 0.3 ) {
                        squares.push_back(approx);
                    }
                }
            }
        }
    }
    return squares;
}

EDIT 17/08/2012:

To draw the detected squares on the image use this code:

cv::Mat debugSquares( std::vector<std::vector<cv::Point> > squares, cv::Mat image )
{
    for ( int i = 0; i< squares.size(); i++ ) {
        // draw contour
        cv::drawContours(image, squares, i, cv::Scalar(255,0,0), 1, 8, std::vector<cv::Vec4i>(), 0, cv::Point());

        // draw bounding rect
        cv::Rect rect = boundingRect(cv::Mat(squares[i]));
        cv::rectangle(image, rect.tl(), rect.br(), cv::Scalar(0,255,0), 2, 8, 0);

        // draw rotated rect
        cv::RotatedRect minRect = minAreaRect(cv::Mat(squares[i]));
        cv::Point2f rect_points[4];
        minRect.points( rect_points );
        for ( int j = 0; j < 4; j++ ) {
            cv::line( image, rect_points[j], rect_points[(j+1)%4], cv::Scalar(0,0,255), 1, 8 ); // blue
        }
    }

    return image;
}
17
  • 1
    Original image can be found here. Jan 14, 2012 at 15:14
  • 1
    I think you can adjust the title of the question for something like Detecting a sheet of paper , if you think it's more appropriate. Jan 14, 2012 at 15:19
  • 1
    @moosgummi I am looking to have same functionality which you have implemented i.e "Detect the Corners of the captured image/document".How you achieved this ? Would I be able to use OpenCV within my iPhone application ? Please suggest me some better way to have this .. Jan 19, 2012 at 7:17
  • 1
    Have you ever done something with OpenCV? Any application at all? Jan 20, 2012 at 11:28
  • 6
    It's worth noting that the flag CV_RETR_EXTERNAL can be used when finding the countours to reject all contours inside a closed shape. Aug 21, 2013 at 16:31

6 Answers 6

172

This is a recurring subject in Stackoverflow and since I was unable to find a relevant implementation I decided to accept the challenge.

I made some modifications to the squares demo present in OpenCV and the resulting C++ code below is able to detect a sheet of paper in the image:

void find_squares(Mat& image, vector<vector<Point> >& squares)
{
    // blur will enhance edge detection
    Mat blurred(image);
    medianBlur(image, blurred, 9);

    Mat gray0(blurred.size(), CV_8U), gray;
    vector<vector<Point> > contours;

    // find squares in every color plane of the image
    for (int c = 0; c < 3; c++)
    {
        int ch[] = {c, 0};
        mixChannels(&blurred, 1, &gray0, 1, ch, 1);

        // try several threshold levels
        const int threshold_level = 2;
        for (int l = 0; l < threshold_level; l++)
        {
            // Use Canny instead of zero threshold level!
            // Canny helps to catch squares with gradient shading
            if (l == 0)
            {
                Canny(gray0, gray, 10, 20, 3); // 

                // Dilate helps to remove potential holes between edge segments
                dilate(gray, gray, Mat(), Point(-1,-1));
            }
            else
            {
                    gray = gray0 >= (l+1) * 255 / threshold_level;
            }

            // Find contours and store them in a list
            findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);

            // Test contours
            vector<Point> approx;
            for (size_t i = 0; i < contours.size(); i++)
            {
                    // approximate contour with accuracy proportional
                    // to the contour perimeter
                    approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true)*0.02, true);

                    // Note: absolute value of an area is used because
                    // area may be positive or negative - in accordance with the
                    // contour orientation
                    if (approx.size() == 4 &&
                            fabs(contourArea(Mat(approx))) > 1000 &&
                            isContourConvex(Mat(approx)))
                    {
                            double maxCosine = 0;

                            for (int j = 2; j < 5; j++)
                            {
                                    double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
                                    maxCosine = MAX(maxCosine, cosine);
                            }

                            if (maxCosine < 0.3)
                                    squares.push_back(approx);
                    }
            }
        }
    }
}

After this procedure is executed, the sheet of paper will be the largest square in vector<vector<Point> >:

opencv paper sheet detection

I'm letting you write the function to find the largest square. ;)

26
  • 4
    That's why I use source control. The smallest accidental modification to the code can be easily discovered. If you didnt change anything, try testing with other images and finally recompile/reinstall opencv. Jan 18, 2012 at 10:39
  • 2
    OpenCV is pretty much the same for all platforms (Win/Linux/Mac/iPhone/...). The difference is that some don't supported the GPU module of OpenCV. Have you built OpenCV for iOS already? Were you able to test it? I think these are the questions you need to answer before trying anything more advanced. Baby steps! Jan 25, 2012 at 12:40
  • 1
    @karlphillip I tested this code and i was able to detect the paper clearly, but it takes so much time. Is the code really heavy? there is an app called SayText where this detection happens in real-time from a video stream. This code would be impractical for real-time, am I right?
    – alandalusi
    Nov 27, 2012 at 17:34
  • 1
    Probably. This is an academic answer, not very practical for the industry. There are all sorts of optimizations you can try, beginning with the definition of the counter located at for (int c = 0; c < 3; c++), which is responsible to iterate on every channel of the image. For instance, you can set it to iterate on only one channel :) Don't forget to up vote. Nov 27, 2012 at 17:45
  • 3
    @SilentPro angle() is a helper function. As stated in the answer, this code is based on samples/cpp/squares.cpp present in OpenCV. Oct 3, 2013 at 0:11
42

Unless there is some other requirement not specified, I would simply convert your color image to grayscale and work with that only (no need to work on the 3 channels, the contrast present is too high already). Also, unless there is some specific problem regarding resizing, I would work with a downscaled version of your images, since they are relatively large and the size adds nothing to the problem being solved. Then, finally, your problem is solved with a median filter, some basic morphological tools, and statistics (mostly for the Otsu thresholding, which is already done for you).

Here is what I obtain with your sample image and some other image with a sheet of paper I found around:

enter image description here enter image description here

The median filter is used to remove minor details from the, now grayscale, image. It will possibly remove thin lines inside the whitish paper, which is good because then you will end with tiny connected components which are easy to discard. After the median, apply a morphological gradient (simply dilation - erosion) and binarize the result by Otsu. The morphological gradient is a good method to keep strong edges, it should be used more. Then, since this gradient will increase the contour width, apply a morphological thinning. Now you can discard small components.

At this point, here is what we have with the right image above (before drawing the blue polygon), the left one is not shown because the only remaining component is the one describing the paper:

enter image description here

Given the examples, now the only issue left is distinguishing between components that look like rectangles and others that do not. This is a matter of determining a ratio between the area of the convex hull containing the shape and the area of its bounding box; the ratio 0.7 works fine for these examples. It might be the case that you also need to discard components that are inside the paper, but not in these examples by using this method (nevertheless, doing this step should be very easy especially because it can be done through OpenCV directly).

For reference, here is a sample code in Mathematica:

f = Import["http://thwartedglamour.files.wordpress.com/2010/06/my-coffee-table-1-sa.jpg"]
f = ImageResize[f, ImageDimensions[f][[1]]/4]
g = MedianFilter[ColorConvert[f, "Grayscale"], 2]
h = DeleteSmallComponents[Thinning[
     Binarize[ImageSubtract[Dilation[g, 1], Erosion[g, 1]]]]]
convexvert = ComponentMeasurements[SelectComponents[
     h, {"ConvexArea", "BoundingBoxArea"}, #1 / #2 > 0.7 &], 
     "ConvexVertices"][[All, 2]]
(* To visualize the blue polygons above: *)
Show[f, Graphics[{EdgeForm[{Blue, Thick}], RGBColor[0, 0, 1, 0.5], 
     Polygon @@ convexvert}]]

If there are more varied situations where the paper's rectangle is not so well defined, or the approach confuses it with other shapes -- these situations could happen due to various reasons, but a common cause is bad image acquisition -- then try combining the pre-processing steps with the work described in the paper "Rectangle Detection based on a Windowed Hough Transform".

11
  • 1
    is there any major difference in implementation of yours and the one above(ie @karlphilip 's answer) ? I am sorry I couldn't find any in a fast look (except 3 channel-1 channel and Mathematica-OpenCV). Feb 15, 2013 at 17:31
  • 2
    @AbidRahmanK yes, there are.. I don't use canny neither "several thresholds" to start with. There are other differences, but by the tone of your comment it seems pointless to put any effort on my own comment.
    – mmgp
    Feb 15, 2013 at 17:52
  • 1
    I see both of you first find the edges, and determine which edge is square. For finding edges, you people use different methods. He uses canny, you use some dilation-erosion. And "several thresholds", may be he got from OpenCV samples, used to find squares. Main thing is, I felt overall concept is same. "Find edges and detect square". And I asked it sincerely, I don't know what "tone" you got from my comment, or what you (understood/misunderstood). So if you feel this question is sincere, I would like to know other differences. Otherwise discard my comments. Feb 15, 2013 at 18:16
  • 1
    @AbidRahmanK of course the concept is the same, the task is the same. Median filtering is being used, thinning is being used, I don't care from where he took several thresholds idea -- it is just not used here (thus how can it not be a difference ?), the image is resized here, the component measurements are different. "Some dilation-erosion" doesn't give binary edges, otsu is used for that. It is pointless to mention this, the code is there.
    – mmgp
    Feb 15, 2013 at 19:02
  • 1
    K. Thank you. Got the answer. Concept is the same. (I never used Mathematica, so I can't understand the code.)And the differences you mentioned are differences, but not a different approach or major ones. If you still didn For example, check this: Feb 16, 2013 at 2:11
20

Well, I'm late.


In your image, the paper is white, while the background is colored. So, it's better to detect the paper is Saturation(饱和度) channel in HSV color space. Take refer to wiki HSL_and_HSV first. Then I'll copy most idea from my answer in this Detect Colored Segment in an image.


Main steps:

  1. Read into BGR
  2. Convert the image from bgr to hsv space
  3. Threshold the S channel
  4. Then find the max external contour(or do Canny, or HoughLines as you like, I choose findContours), approx to get the corners.

This is my result:

enter image description here


The Python code(Python 3.5 + OpenCV 3.3):

#!/usr/bin/python3
# 2017.12.20 10:47:28 CST
# 2017.12.20 11:29:30 CST

import cv2
import numpy as np

##(1) read into  bgr-space
img = cv2.imread("test2.jpg")

##(2) convert to hsv-space, then split the channels
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
h,s,v = cv2.split(hsv)

##(3) threshold the S channel using adaptive method(`THRESH_OTSU`) or fixed thresh
th, threshed = cv2.threshold(s, 50, 255, cv2.THRESH_BINARY_INV)

##(4) find all the external contours on the threshed S
#_, cnts, _ = cv2.findContours(threshed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cv2.findContours(threshed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]

canvas  = img.copy()
#cv2.drawContours(canvas, cnts, -1, (0,255,0), 1)

## sort and choose the largest contour
cnts = sorted(cnts, key = cv2.contourArea)
cnt = cnts[-1]

## approx the contour, so the get the corner points
arclen = cv2.arcLength(cnt, True)
approx = cv2.approxPolyDP(cnt, 0.02* arclen, True)
cv2.drawContours(canvas, [cnt], -1, (255,0,0), 1, cv2.LINE_AA)
cv2.drawContours(canvas, [approx], -1, (0, 0, 255), 1, cv2.LINE_AA)

## Ok, you can see the result as tag(6)
cv2.imwrite("detected.png", canvas)

Related answers:

  1. How to detect colored patches in an image using OpenCV?
  2. Edge detection on colored background using OpenCV
  3. OpenCV C++/Obj-C: Detecting a sheet of paper / Square Detection
  4. How to use `cv2.findContours` in different OpenCV versions?
1
3

What you need is a quadrangle instead of a rotated rectangle. RotatedRect will give you incorrect results. Also you will need a perspective projection.

Basicly what must been done is:

  • Loop through all polygon segments and connect those which are almost equel.
  • Sort them so you have the 4 most largest line segments.
  • Intersect those lines and you have the 4 most likely corner points.
  • Transform the matrix over the perspective gathered from the corner points and the aspect ratio of the known object.

I implemented a class Quadrangle which takes care of contour to quadrangle conversion and will also transform it over the right perspective.

See a working implementation here: Java OpenCV deskewing a contour

1

Once you have detected the bounding box of the document, you can perform a four-point perspective transform to obtain a top-down birds eye view of the image. This will fix the skew and isolate only the desired object.


Input image:

Detected text object

Top-down view of text document

Code

from imutils.perspective import four_point_transform
import cv2
import numpy

# Load image, grayscale, Gaussian blur, Otsu's threshold
image = cv2.imread("1.png")
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (7,7), 0)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]

# Find contours and sort for largest contour
cnts = cv2.findContours(thresh, cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)
displayCnt = None

for c in cnts:
    # Perform contour approximation
    peri = cv2.arcLength(c, True)
    approx = cv2.approxPolyDP(c, 0.02 * peri, True)
    if len(approx) == 4:
        displayCnt = approx
        break

# Obtain birds' eye view of image
warped = four_point_transform(image, displayCnt.reshape(4, 2))

cv2.imshow("thresh", thresh)
cv2.imshow("warped", warped)
cv2.imshow("image", image)
cv2.waitKey()
2
  • Can you please provide c++ implementation of above code? Nov 8, 2020 at 12:42
  • This is exactly the one I needed! but i need it in Objectice-C
    – Olufsen
    Dec 27, 2020 at 8:56
-1

Detecting sheet of paper is kinda old school. If you want to tackle skew detection then it is better if you straightaway aim for text line detection. With this you will get the extremas left, right, top and bottom. Discard any graphics in the image if you dont want and then do some statistics on the text line segments to find the most occurring angle range or rather angle. This is how you will narrow down to a good skew angle. Now after this you put these parameters the skew angle and the extremas to deskew and chop the image to what is required.

As for the current image requirement, it is better if you try CV_RETR_EXTERNAL instead of CV_RETR_LIST.

Another method of detecting edges is to train a random forests classifier on the paper edges and then use the classifier to get the edge Map. This is by far a robust method but requires training and time.

Random forests will work with low contrast difference scenarios for example white paper on roughly white background.

Not the answer you're looking for? Browse other questions tagged or ask your own question.