1

I am working on Android+OpenCv+JNI to find out the largest contour in the image and then cropping that largest Contour using perspective transformation. My problem is after applying transformation I am not able to convert resultant Mat to Bitmap and returning with error

OpenCV Error: Assertion failed (src.type() == CV_8UC1 || src.type() == CV_8UC3 || src.type() == CV_8UC4) in void Java_org_opencv_android_Utils_nMatToBitmap2(JNIEnv*, jclass, jlong, jobject, jboolean), file /home/reports/ci/slave_desktop/50-SDK/opencv/modules/java/generator/src/cpp/utils.cpp, line 98

Here is my JNI code:

JNIEXPORT jint JNICALL Java_org_opencv_samples_tutorial3_Sample3Native_FindSquares(

    JNIEnv* env, jobject, jlong addrRgba, jint draw, jlong addrDescriptor) {

Mat& image = *(Mat*) addrRgba;
Mat& pMatDesc = *(Mat*) addrDescriptor;
int thresh = 50, N = 4;
int found = 0;

Mat pyr, timg, gray0(image.size(), CV_8U), gray;

// down-scale and upscale the image to filter out the noise
pyrDown(image, pyr, Size(image.cols / 2, image.rows / 2));
pyrUp(pyr, timg, image.size());
vector < vector<Point> > contours;
// find squares in every color plane of the image
for (int c = 1; c < 3; c++) {
    int ch[] = { c, 0 };
    mixChannels(&timg, 1, &gray0, 1, ch, 1);
    // try several threshold levels
    for (int l = 0; l < N; l++) {
        // hack: use Canny instead of zero threshold level.
        // Canny helps to catch squares with gradient shading
        if (l == 0) {
            // apply Canny. Take the upper threshold from slider
            // and set the lower to 0 (which forces edges merging)
            Canny(gray0, gray, 0, thresh, 5);
            // dilate canny output to remove potential
            // holes between edge segments
            dilate(gray, gray, Mat(), Point(-1, -1));
        } else {
            // apply threshold if l!=0:
            //     tgray(x,y) = gray(x,y) < (l+1)*255/N ? 255 : 0
            gray = gray0 >= (l + 1) * 255 / N;
        }
        // find contours and store them all as a list
        findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);
        vector<Point> approx;
        // test each contour
        for (size_t i = 0; i < contours.size(); i++) {

            //__android_log_print(ANDROID_LOG_INFO, "Test", "Error:", v);
            // approximate contour with accuracy proportional
            // to the contour perimeter
            approxPolyDP(Mat(contours[i]), approx,
                    arcLength(Mat(contours[i]), true) * 0.02, true);

            // square contours should have 4 vertices after approximation
            // relatively large area (to filter out noisy contours)
            // and be convex.
            // Note: absolute value of an area is used because
            // area may be positive or negative - in accordance with the
            // contour orientation
            if (approx.size() == 4 && fabs(contourArea(Mat(approx))) > 1000
                    && isContourConvex(Mat(approx))) {
                double maxCosine = 0;

                for (int j = 2; j < 5; j++) {
                    // find the maximum cosine of the angle between joint edges
                    double cosine = fabs(
                            angle(approx[j % 4], approx[j - 2],
                                    approx[j - 1]));
                    maxCosine = MAX(maxCosine, cosine);
                }

                // if cosines of all angles are small
                // (all angles are ~90 degree) then write quandrange
                // vertices to resultant sequence
                if (maxCosine < 0.3) {

                    circle(image, approx[0], 5, Scalar(255, 0, 0, 255), 3,
                            4, 0);
                    circle(image, approx[1], 5, Scalar(255, 0, 0, 255), 3,
                            4, 0);
                    circle(image, approx[2], 5, Scalar(255, 0, 0, 255), 3,
                            4, 0);
                    circle(image, approx[3], 5, Scalar(255, 0, 0, 255), 3,
                            4, 0);
                    //rectangle(image, approx[0], approx[2], Scalar(0,255,0,255), 5, 4, 0);

                    //Center of this rectangle
                    int x = (int) ((approx[0].x + approx[1].x + approx[2].x
                            + approx[3].x) / 4.0);
                    int y = (int) ((approx[0].y + approx[1].y + approx[2].y
                            + approx[3].y) / 4.0);

                    if ((int) draw) {
                        //outline
                        line(image, approx[0], approx[1],
                                Scalar(0, 255, 0, 255), 1, 4, 0);
                        line(image, approx[1], approx[2],
                                Scalar(0, 255, 0, 255), 1, 4, 0);
                        line(image, approx[2], approx[3],
                                Scalar(0, 255, 0, 255), 1, 4, 0);
                        line(image, approx[3], approx[0],
                                Scalar(0, 255, 0, 255), 1, 4, 0);
                        //center
                        //circle(image, Point(x,y), 1, Scalar(255,0,0,255));
                    }
                    vector<Point2f> src(4);
                    src[0] = approx[0];
                    src[1] = approx[1];
                    src[2] = approx[2];
                    src[3] = approx[3];
                    cv::Mat quad = cv::Mat::zeros(300, 220, CV_32FC1 );

                    // transformed quadrangle
                    vector<Point2f> quad_pts(4);


                      quad_pts[0] = Point(0, 0);
                      quad_pts[1] = Point(quad.cols, 0);
                      quad_pts[2] = Point(quad.cols, quad.rows);
                      quad_pts[3] = Point(0, quad.rows);

                    Mat transmtx = getPerspectiveTransform(src, quad_pts);
                    warpPerspective(src, quad, transmtx, quad.size());

                    quad.copyTo(pMatDesc);
                    found = 1;
                    jint result = (jint) found;
                    return result;
                }
            }
        }
    }
}
jint result = (jint) found;
return result;

}

In my java code I am calling this function as

found = FindSquares(mRgba.getNativeObjAddr(), mDraw, descriptor.getNativeObjAddr());

And finally I am trying to convert final Mat to Bitmap

Mat final_mat = new Mat(descriptor.height(), descriptor.width(), CvType.CV_8UC4);
descriptor.copyTo(final_mat); 
bitmap = Bitmap.createBitmap(final_mat.cols(), final_mat.rows(),
                Bitmap.Config.ARGB_8888);
Utils.matToBitmap(final_mat, bitmap);

The final_mat channel type is becoming CV_32FC1. How to convert the channel type to CV_8UC4 Please help me to find out the solution.

Edit: I have changed the finat_mat image to CV_8UC3

descriptor.copyTo(final_mat);
descriptor.convertTo(final_mat, CvType.CV_8UC1);
Imgproc.cvtColor(final_mat,final_mat,Imgproc.COLOR_GRAY2RGB);

But I am getting blank(Black) image as a result??

1 Answer 1

1

After a long research I've found the solution. Here the problem raised because of conversions on actual image. We should use duplicate of actual Mat object to apply conversions(blurred, canny etc..), and use actual Mat object with the warp perspective transformation. Here I am attaching the reference code to find out the Largest Contour.

jni_part.cpp:

extern "C" {
double angle(Point pt1, Point pt2, Point pt0);

JNIEXPORT jint Java_info_androidhive_androidcameraapi_CameraMainActivity_findSquare(
    JNIEnv*, jobject, jlong addrRgba, jlong addrDescriptor, jint width_,
    jint height_);

JNIEXPORT jint Java_info_androidhive_androidcameraapi_CameraMainActivity_findSquare(
    JNIEnv*, jobject, jlong addrRgba, jlong addrDescriptor, jint width_,
    jint height_) {

Mat& image = *(Mat*) addrRgba;
Mat& imageCropped = *(Mat*) addrDescriptor;
int screen_width = (int) width_;
int screen_height = (int) height_;

Mat newSrc = image.clone();
imageCropped = image.clone();
Mat testImage = image.clone();
// blur will enhance edge detection
Mat blurred(testImage);

medianBlur(testImage, blurred, 9);

Mat gray0(blurred.size(), CV_8U), gray;
vector<vector<Point> > contours;

// find squares in every color plane of the image
cv::vector<cv::vector<cv::Point> > squares;

for (int c = 0; c < 3; c++) {
    int ch[] = { c, 0 };
    mixChannels(&blurred, 1, &gray0, 1, ch, 1);

    // try several threshold levels
    const int threshold_level = 2;
    for (int l = 0; l < threshold_level; l++) {
        // Use Canny instead of zero threshold level!
        // Canny helps to catch squares with gradient shading
        if (l == 0) {
            Canny(gray0, gray, 10, 20, 3); //

            // Dilate helps to remove potential holes between edge segments
            dilate(gray, gray, Mat(), Point(-1, -1));
        } else {
            gray = gray0 >= (l + 1) * 255 / threshold_level;
        }

        // Find contours and store them in a list
        findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);

        // Test contours
        vector<Point> approx;
        if (contours.size() > 0) {
            for (size_t i = 0; i < contours.size(); i++) {
                // approximate contour with accuracy proportional
                // to the contour perimeter
                approxPolyDP(Mat(contours[i]), approx,
                        arcLength(Mat(contours[i]), true) * 0.02, true);

                // Note: absolute value of an area is used because
                // area may be positive or negative - in accordance with the
                // contour orientation
                if (approx.size() == 4
                        && fabs(contourArea(Mat(approx))) > 1000
                        && isContourConvex(Mat(approx))) {
                    double maxCosine = 0;

                    for (int j = 2; j < 5; j++) {
                        double cosine = fabs(
                                angle(approx[j % 4], approx[j - 2],
                                        approx[j - 1]));
                        maxCosine = MAX(maxCosine, cosine);
                    }

                    if (maxCosine < 0.3) {
                        squares.push_back(approx);

                        /*circle(image, approx[0], 5, Scalar(255, 0, 0, 255), 3,
                         4, 0);
                         circle(image, approx[1], 5, Scalar(255, 0, 0, 255), 3,
                         4, 0);
                         circle(image, approx[2], 5, Scalar(255, 0, 0, 255), 3,
                         4, 0);
                         circle(image, approx[3], 5, Scalar(255, 0, 0, 255), 3,
                         4, 0);
                         if ((int) draw) {
                         line(image, approx[0], approx[1],
                         Scalar(0, 255, 0, 255), 2, 4, 0);
                         line(image, approx[1], approx[2],
                         Scalar(0, 255, 0, 255), 2, 4, 0);
                         line(image, approx[2], approx[3],
                         Scalar(0, 255, 0, 255), 2, 4, 0);
                         line(image, approx[3], approx[0],
                         Scalar(0, 255, 0, 255), 2, 4, 0);
                         }*/
                    }
                }
            }
        }
    }
}
if (squares.size() > 0) {
    int max_width = 0;
    int max_height = 0;
    int max_square_idx = 0;
    cv::vector<cv::Point> biggest_square;

            squares.size());
    for (size_t i = 0; i < squares.size(); i++) {

    cv::Rect structure.
        cv::Rect rectangle = boundingRect(cv::Mat(squares[i]));
        // Store the index position of the biggest square found
        if ((rectangle.width >= max_width)
                && (rectangle.height >= max_height)) {
            max_width = rectangle.width;
            max_height = rectangle.height;
            max_square_idx = i;
        }
    }

    biggest_square = squares[max_square_idx];
    vector<Point> _adjustRect;
    _adjustRect = squares[max_square_idx];
    if (biggest_square.size() == 4) {
        vector<Point> sortedPoints;
        sortedPoints = squares[max_square_idx];

        Point ptbiggest_square = biggest_square[0];

        Point ptBottomLeft1 = biggest_square[0];
        Point ptBottomRight1 = biggest_square[1];
        Point ptTopRight1 = biggest_square[2];
        Point ptTopLeft1 = biggest_square[3];

        int bl = ptBottomLeft1.x + ptBottomLeft1.y;
        int br = ptBottomRight1.x + ptBottomRight1.y;
        int tr = ptTopRight1.x + ptTopRight1.y;
        int tl = ptTopLeft1.x + ptTopLeft1.y;

        int value_array[] = { bl, br, tr, tl };
        int max = value_array[0];
        int min = value_array[0];

        for (int s = 0; s < 4; s++) {
            if (value_array[s] > max) {
                max = value_array[s];
            } else if (value_array[s] < min) {
                min = value_array[s];
            }
        }
        int minIndex = 0;
        int maxIndex = 0;

        int missingIndexOne = 0;
        int missingIndexTwo = 0;

        for (int i = 0; i < 4; i++) {

            if (value_array[i] == min) {
                sortedPoints[0] = biggest_square[i];
                minIndex = i;
                continue;
            }

            if (value_array[i] == max) {
                sortedPoints[2] = biggest_square[i];
                maxIndex = i;
                continue;
            }
            missingIndexOne = i;
        }

        for (int i = 0; i < 4; i++) {
            if (missingIndexOne != i && minIndex != i && maxIndex != i) {
                missingIndexTwo = i;
            }
        }

        if (biggest_square[missingIndexOne].x
                < biggest_square[missingIndexTwo].x) {
            //2nd Point Found

            sortedPoints[3] = biggest_square[missingIndexOne];
            sortedPoints[1] = biggest_square[missingIndexTwo];
        } else {
            //4rd Point Found

            sortedPoints[1] = biggest_square[missingIndexOne];
            sortedPoints[3] = biggest_square[missingIndexTwo];
        }

        _adjustRect[0] = sortedPoints[0];
        _adjustRect[1] = sortedPoints[1];
        _adjustRect[2] = sortedPoints[2];
        _adjustRect[3] = sortedPoints[3];



    }

    Point ptTopLeft = _adjustRect[0];
    Point ptTopRight = _adjustRect[1];
    Point ptBottomRight = _adjustRect[2];
    Point ptBottomLeft = _adjustRect[3];

    float imageScale = fminf((float) screen_width / newSrc.cols,
            (float) screen_height / newSrc.rows);

    __android_log_print(ANDROID_LOG_INFO, "OpenGLTest", "imageScale %f",
            imageScale);
    __android_log_print(ANDROID_LOG_INFO, "OpenGLTest", "width_ %d",
            screen_width);

    float w1 = sqrt(
            pow(ptBottomRight.x / imageScale - ptBottomLeft.x / imageScale,
                    2)
                    + pow(
                            ptBottomRight.x / imageScale
                                    - ptBottomLeft.x / imageScale, 2));
    float w2 = sqrt(
            pow(ptTopRight.x / imageScale - ptTopLeft.x / imageScale, 2)
                    + pow(
                            ptTopRight.x / imageScale
                                    - ptTopLeft.x / imageScale, 2));

    float h1 = sqrt(
            pow(ptTopRight.y / imageScale - ptBottomRight.y / imageScale, 2)
                    + pow(
                            ptTopRight.y / imageScale
                                    - ptBottomRight.y / imageScale, 2));
    float h2 = sqrt(
            pow(ptTopLeft.y / imageScale - ptBottomLeft.y / imageScale, 2)
                    + pow(
                            ptTopLeft.y / imageScale
                                    - ptBottomLeft.y / imageScale, 2));

    float maxWidth = (w1 < w2) ? w1 : w2;
    float maxHeight = (h1 < h2) ? h1 : h2;

    Point2f src[4], quad[4];
    src[0].x = ptTopLeft.x;
    src[0].y = ptTopLeft.y;
    src[1].x = ptTopRight.x;
    src[1].y = ptTopRight.y;
    src[2].x = ptBottomRight.x;
    src[2].y = ptBottomRight.y;
    src[3].x = ptBottomLeft.x;
    src[3].y = ptBottomLeft.y;

    quad[0].x = 0;
    quad[0].y = 0;
    quad[1].x = maxWidth - 1;
    quad[1].y = 0;
    quad[2].x = maxWidth - 1;
    quad[2].y = maxHeight - 1;
    quad[3].x = 0;
    quad[3].y = maxHeight - 1;

    cv::Mat undistorted = cv::Mat(cvSize(maxWidth, maxHeight), CV_8UC1);
    cv::warpPerspective(newSrc, undistorted,
            cv::getPerspectiveTransform(src, quad),
            cvSize(maxWidth, maxHeight));

    imageCropped = undistorted.clone();
}

return 1;

}

double angle(Point pt1, Point pt2, Point pt0) {
double dx1 = pt1.x - pt0.x;
double dy1 = pt1.y - pt0.y;
double dx2 = pt2.x - pt0.x;
double dy2 = pt2.y - pt0.y;
return (dx1 * dx2 + dy1 * dy2)
        / sqrt((dx1 * dx1 + dy1 * dy1) * (dx2 * dx2 + dy2 * dy2) + 1e-10);
}

}

Happy Coding!!

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.