JavaCV Perspective Correction

Question

I converted the perspective correction code implemented using OpenCV and C++ at: https://opencv-code.com/tutorials/automatic-perspective-correction-for-quadrilateral-objects/

to obtain the following OpenCV code implemented in Java:

public class project 
{
static Point2f center;
public static void main(String args[])
{
    System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
    center = new Point2f(0,0);
    Mat src = new Mat();
    src = Highgui.imread("image.jpg");
    if(src == null)
    {
        System.out.println("Image not loaded");
        System.exit(1);
    }

    Mat bw = new Mat();

    Imgproc.cvtColor(src, bw, Imgproc.COLOR_BGR2GRAY);
    Imgproc.blur(bw, bw, new Size(3,3));
    Imgproc.Canny(bw, bw, 100, 100, 3,true);

     Mat lines = new Mat();
        int threshold = 70;
        int minLineSize = 30;
        int lineGap = 10;

        Imgproc.HoughLinesP(bw, lines, 1, Math.PI / 180, threshold,
                minLineSize, lineGap);

        for (int x = 0; x < lines.cols(); x++) 
        {

            double[] vec = lines.get(0, x);
            double[] val = new double[4];

            val[0] = 0;
            val[1] = ((float) vec[1] - vec[3]) / (vec[0] - vec[2]) * -vec[0] + vec[1];
            val[2] = src.cols();
            val[3] = ((float) vec[1] - vec[3]) / (vec[0] - vec[2]) * (src.cols() - vec[2]) + vec[3];

            lines.put(0, x, val);

        }

        List<Point2f> corners = new ArrayList<Point2f>();
        for (int i = 0; i < lines.cols(); i++)
        {
            for (int j = i+1; j < lines.cols(); j++)
            {
                Mat m1 = null,m2 = null;
                double[] d1 = lines.get(0,i);
                double[] d2 = lines.get(0, j);
                m1.put(0, i, d1);
                m2.put(0, j, d2);
                Point2f pt = computeIntersect(m1, m2);
                if (pt.x >= 0 && pt.y >= 0)
                    corners.add(pt);
            }
        }

        List<Point2f> approx = new ArrayList<Point2f>();
        List<Point2f> curve;
        MatOfPoint2f mat2f = new MatOfPoint2f();
        for(int k=0;k<corners.size();++k)
        {
            Point2f rec = corners.get(k);
            Point p = new Point(rec.x,rec.y);
            mat2f.fromArray(p);
        }
        MatOfPoint2f mat2frec = new MatOfPoint2f();
        Imgproc.approxPolyDP(mat2f, mat2frec, Imgproc.arcLength(mat2f, true) * 0.02,true);

        if (approx.size() != 4)
        {
            System.out.println("The object is not quadrilateral!");

        }

        // Get mass center
        for (int i = 0; i < corners.size(); i++)
        {
            center.x = center.x + corners.get(i).x;
            center.y = center.y + corners.get(i).y;
        }
        center.x *= (1. / corners.size());
        center.y *= (1. / corners.size());

        sortCorners(corners, center);

        Mat dst = src.clone();

        // Draw lines
        for (int i = 0; i < lines.cols(); i++)
        {
            double[] v = lines.get(0, i);
            Scalar cc = new Scalar(0,255,0,0);
            Core.line(dst, new Point(v[0], v[1]), new Point(v[2], v[3]), cc);

        }

        Scalar c1 = new Scalar(0,0,255,0);
        Scalar c2 = new Scalar(0,255,0,0);
        Scalar c3 = new Scalar(255,0,0,0);
        Scalar c4 = new Scalar(255,255,255,0);

        // Draw corner points

        Core.circle(dst, new Point(corners.get(0).x,corners.get(0).y), 3, c1, 2);
        Core.circle(dst, new Point(corners.get(1).x,corners.get(1).y), 3, c2, 2);
        Core.circle(dst, new Point(corners.get(2).x,corners.get(2).y), 3, c3, 2);
        Core.circle(dst, new Point(corners.get(3).x,corners.get(3).y), 3, c4, 2);

        Scalar c5 = new Scalar(0,255,255,0);
        // Draw mass center
        Core.circle(dst, new Point(center.x,center.y), 3, c5, 2);

        Mat quad = Mat.zeros(300, 220, CvType.CV_8UC3);

        List<Point2f> quad_pts = new ArrayList<Point2f>();
        quad_pts.add(new Point2f(0, 0));
        quad_pts.add(new Point2f(quad.cols(), 0));
        quad_pts.add(new Point2f(quad.cols(), quad.rows()));
        quad_pts.add(new Point2f(0, quad.rows()));


        Mat transmtx = Imgproc.getPerspectiveTransform((Mat) corners, (Mat) quad_pts);
        Imgproc.warpPerspective(src, quad, transmtx, quad.size());

        MatOfByte matOfByte = new MatOfByte();

        Highgui.imencode(".jpg", dst, matOfByte); 
        byte[] byteArray = matOfByte.toArray();
        BufferedImage bufImage = null;
        try 
        {
            InputStream in = new ByteArrayInputStream(byteArray);
            bufImage = ImageIO.read(in);
            File outputfile = new File("Image.jpg");
            ImageIO.write(bufImage, "jpg", outputfile);
        } 
        catch (Exception e) {
            e.printStackTrace();
        }

        MatOfByte matOfByte2 = new MatOfByte();

        Highgui.imencode(".jpg", dst, matOfByte2); 
        byte[] byteArray2 = matOfByte2.toArray();
        BufferedImage bufImage2 = null;
        try 
        {
            InputStream in = new ByteArrayInputStream(byteArray2);
            bufImage2 = ImageIO.read(in);
            File outputfile2 = new File("Quadrilateral.jpg");
            ImageIO.write(bufImage, "jpg", outputfile2);
        } 
        catch (Exception e) {
            e.printStackTrace();
        }


}
static Point2f computeIntersect(Mat es, Mat es2)
{
    int size = (int) es.total() * es.channels();
    float[] buff = new float[size];
    es.get(0, 0, buff);

    int size1 = (int) es.total() * es.channels();
    float[] buff1 = new float[size1];
    es.get(0, 0, buff1);

    float x1=buff[0], y1 = buff[1], x2 = buff[2], y2 = buff[3];
    float x3 = buff1[0], y3 = buff1[1], x4 = buff1[2], y4 = buff1[3];
    float denom;
    float d;
    d = (Float) null;
    d = (float)((x1 - x2) * (y3 - y4)) - ((y1 - y2) * (x3 - x4));
    if (d != (Float) null)
    {
        Point2f pt = new Point2f();
        pt.x = ((x1 * y2 - y1 * x2) * (x3 - x4) - (x1 - x2) * (x3 * y4 - y3 * x4)) / d;
        pt.y = ((x1 * y2 - y1 * x2) * (y3 - y4) - (y1 - y2) * (x3 * y4 - y3 * x4)) / d;
        return pt;
    }
    else
        return new Point2f(-1, -1);
}

static void sortCorners(List<Point2f> corners,Point2f center)
{
    List<Point2f> top = null, bot = null;

    for (int i = 0; i < corners.size(); i++)
    {
        if (corners.get(i).y < center.y)
            top.add(corners.get(i));
        else
            bot.add(corners.get(i));
    }

    Point2f tl = top.get(0).x > top.get(1).x ? top.get(1) : top.get(0);
    Point2f tr = top.get(0).x > top.get(1).x ? top.get(0) : top.get(1);
    Point2f bl = bot.get(0).x > bot.get(1).x ? bot.get(1) : bot.get(0);
    Point2f br = bot.get(0).x > bot.get(1).x ? bot.get(0) : bot.get(1);

    corners.clear();
    corners.add(tl);
    corners.add(tr);
    corners.add(br);
    corners.add(bl);
}



}

I'm having trouble converting List< Point2f > to MatOfPoint2f. The arcLength(..) function is therefore not working and the code doesn't seem to work. I'm hoping someone can help.

Answer 1

This a part of the implementation that i used in my project.I already had the exact corner points using an algo i developed but the rest is given in this code.Do not use point2fs. Use point arrays and them convert them into matofpoint2fs.

the jarfile containing Imshow can be downloaded from here.It is very effective in testing your o/p at any point of time. Add this package to your program: https://github.com/master-atul/ImShow-Java-OpenCV

Details regarding approxpolydp: http://docs.opencv.org/java/org/opencv/imgproc/Imgproc.html#approxPolyDP%28org.opencv.core.MatOfPoint2f,org.opencv.core.MatOfPoint2f,double,boolean%29

And u don't have to use arclength. Just give an approx value for epsilon depending on the clarity of your input.(like 2.0 or 3.0..)

(sort is the function used to sort the corners).

int a[][],imgarr[][];

Point p[];

BufferedImage img;

int w,h;
void sort()
{
int x = (a[0][0] + a[1][0] + a[2][0] + a[3][0])/4;
int y = (a[0][1] + a[1][1] + a[2][1] + a[3][1])/4;
int j = 0;
int order[] = new int[4];
double tans[] = new double[4];
double tans1[] = new double[4];
int tmpar[][] = new int[4][2];
p = new Point[4];       
for(int i = 0;i<4;i++)
{
tans1[i] = tans[i] = Math.atan2(a[i][1] - y , a[i][0] - x);//finding angles for sorting corners 
}
Arrays.sort(tans1);
for(int i = 0;i<2;i++)
{
double temp = tans1[i];
tans1[i]= tans1[3-i];
tans1[3-i] = temp;
}
for(int i=0;i<4;i++)
{
for(j = 0;j<4;j++)
{
    if(tans1[i]==tans[j])
        break;
}
order[i] = j;
}
for(int i = 0;i<4;i++)
{
for(j=0;j<2;j++)
{
    tmpar[i][j] = a[i][j];
}
}
for(int i = 0;i<4;i++)
{
for(j = 0;j<2;j++)
{
        a[i][j] = tmpar[order[i]][j];
        }
    }
    p[0] = new Point(a[0][1],a[0][0]);
    p[1] = new Point(a[1][1],a[1][0]);
    p[2] = new Point(a[2][1],a[2][0]);
    p[3] = new Point(a[3][1],a[3][0]);
}
void transform() throws Exception
{
    Point farray[] = new Point[4];      
    try
    {
        img = ImageIO.read(new File("C:/Users/Documents/a.jpg"));
    }
    catch(Exception r)
    {
        System.out.println("no file");
    }
    PixelGrabber pg;
    if(img==null)
    {
        return;
    }
    w = img.getWidth();
    h = img.getHeight();
    imgarr = new int[h][w];
    try
    {           
        for(int i = 0; i < h ; i++)
        {
            pg = new PixelGrabber(img,0,i,w,1,imgarr[i],0,w);               
            pg.grabPixels();    
        }
        changeto256();
    }
    catch(Exception e)
    {
        System.out.println("here "+e);
    }
    int m=0;        
    byte array[] = new byte[w*h];
    int iar[] = new int[w*h];
    for(int i = 0 ; i < h ; i++)
    {
        for(int j = 0 ; j < w ; j++)
        {
            array[m++]= (byte)imgarr[i][j];
        }
    }
    farray[3] = new Point(0,0);
    farray[0] = new Point(w,0);
    farray[1] = new Point(w,h);
    farray[2] = new Point(0,h);


    Mat mat = new Mat(h,w, CvType.CV_8U);
    mat.put(0, 0, array);
    Imshow is = new Imshow("try");
    MatOfPoint2f quad = new MatOfPoint2f(p);
    MatOfPoint2f rect = new MatOfPoint2f(farray);
    Mat transmtx = Imgproc.getPerspectiveTransform(quad,rect);
    Mat output = new Mat(w,h,CvType.CV_8U); 
    Imgproc.warpPerspective(mat, output, transmtx, new size(w,h),Imgproc.INTER_CUBIC);
    is.showImage(output);       
    MatOfByte matOfByte = new MatOfByte();
    Highgui.imencode(".jpg", output, matOfByte); 
    byte[] byteArray = matOfByte.toArray();
    File f = new File("retrieve1.jpg");
    BufferedImage img1 =null;
    InputStream in = new ByteArrayInputStream(byteArray);
    img1  = ImageIO.read(in);
    WritableRaster raster = (WritableRaster)img1.getData();
    raster.setDataElements(0,0,byteArray);
    img1.setData(raster);
    try
    {
        ImageIO.write(img1,"jpg",f);
    }
    catch(Exception e)
    {}
}