OpenCV&OpenGL开发AR的实现

ti

这篇文章介绍OpenCV&OpenGL开发AR的介绍。主要涉及到的内容看下图我打开的包的内容。首先和第四章中对比一下，发现多了ARCubeRenderer类，adapters包，ARFilter、NoneARFilter接口类。

我们还是从主类看起：

/**
* 这里使用Framelayout布局管理器，然后添加用于显示实时获取的视频帧的CameraView类，
* 再添加用于渲染三维虚拟模型的GLSurfaceView类
*/
FrameLayout layout = new FrameLayout(this);
layout.setLayoutParams(new FrameLayout.LayoutParams(
FrameLayout.LayoutParams.MATCH_PARENT,
FrameLayout.LayoutParams.MATCH_PARENT));
setContentView(layout);

mCameraView = new NativeCameraView(this, mCameraIndex);
mCameraView.setCvCameraViewListener(this);
mCameraView.setLayoutParams(new FrameLayout.LayoutParams(
FrameLayout.LayoutParams.MATCH_PARENT,
FrameLayout.LayoutParams.MATCH_PARENT));
layout.addView(mCameraView);

GLSurfaceView glSurfaceView = new GLSurfaceView(this);
glSurfaceView.getHolder().setFormat(
PixelFormat.TRANSPARENT);
glSurfaceView.setEGLConfigChooser(8, 8, 8, 8, 0, 0);
glSurfaceView.setZOrderOnTop(true);
glSurfaceView.setLayoutParams(new FrameLayout.LayoutParams(
FrameLayout.LayoutParams.MATCH_PARENT,
FrameLayout.LayoutParams.MATCH_PARENT));
layout.addView(glSurfaceView);

/**
* 关于计算机图形学中摄像机模型的一些内容包括摄像机内部固有参数，OpenGL视锥模型
* 可以参考计算机图形中的有关知识以及小孔成像模型
*/
mCameraProjectionAdapter = new CameraProjectionAdapter();

mARRenderer = new ARCubeRenderer();
mARRenderer.cameraProjectionAdapter =
mCameraProjectionAdapter;
glSurfaceView.setRenderer(mARRenderer);

再来看CameraProjectionAdapter类：

/**
* 关于计算机图形学中摄像机模型的一些内容
* 包括摄像机内部固有参数，OpenGL视锥模型
* 可以参考计算机图形中的有关知识以及小孔成像模型
* @author scy
*
*/
public class CameraProjectionAdapter {

float mFOVY = 43.6f; // 30mm equivalent
float mFOVX = 65.4f; // 30mm equivalent
int mHeightPx = 640;
int mWidthPx = 480;
float mNear = 1f;
float mFar = 10000f;

final float[] mProjectionGL = new float[16];
boolean mProjectionDirtyGL = true;

MatOfDouble mProjectionCV;
boolean mProjectionDirtyCV = true;

// 关于摄像头参数的获取
// 可以通过摄像头标定，也可以像这里通过AndroidSDK获取
// 如果能够实时对摄像头进行标定最好，如果不能实时的话，离线标定就需要对不同的移动终端进行标定，这样效率会比较低，而且怎么用于商业应用呢？
// 还是这种比较理想，不知道高通是怎么做的？有知道朋友可以告诉我一下，谢谢！
public void setCameraParameters(Parameters parameters) {
mFOVY = parameters.getVerticalViewAngle();
mFOVX = parameters.getHorizontalViewAngle();

Size pictureSize = parameters.getPictureSize();
mHeightPx = pictureSize.height;
mWidthPx = pictureSize.width;

mProjectionDirtyGL = true;
mProjectionDirtyCV = true;
}

public void setClipDistances(float near, float far) {
mNear = near;
mFar = far;
mProjectionDirtyGL = true;
}

// http://blog.csdn.net/lyx2007825/article/details/8792475
// FOV为视野角，这里有水平和垂直两种，见附图所示：
public float[] getProjectionGL() {
if (mProjectionDirtyGL) {
//
final float top =
(float)Math.tan(mFOVY * Math.PI / 360f) * mNear;
final float right =
(float)Math.tan(mFOVX * Math.PI / 360f) * mNear;
// 跟据设备屏幕的几何特征创建投影矩阵
Matrix.frustumM(mProjectionGL, 0,
-right, right, -top, top, mNear, mFar);
mProjectionDirtyGL = false;
}
return mProjectionGL;
}

// 获取计算机视觉坐标中的投影矩阵
// 跟摄像头的内部固有参数
public MatOfDouble getProjectionCV() {
if (mProjectionDirtyCV) {
if (mProjectionCV == null) {
mProjectionCV = new MatOfDouble();
mProjectionCV.create(3, 3, CvType.CV_64FC1);
}

double diagonalPx = Math.sqrt(
(Math.pow(mWidthPx, 2.0) +
Math.pow(mHeightPx, 2.0)));
double diagonalFOV = Math.sqrt(
(Math.pow(mFOVX, 2.0) +
Math.pow(mFOVY, 2.0)));
double focalLengthPx = diagonalPx /
(2.0 * Math.tan(0.5 * diagonalFOV));

mProjectionCV.put(0, 0, focalLengthPx);
mProjectionCV.put(0, 1, 0.0);
mProjectionCV.put(0, 2, 0.5 * mWidthPx);
mProjectionCV.put(1, 0, 0.0);
mProjectionCV.put(1, 1, focalLengthPx);
mProjectionCV.put(1, 2, 0.5 * mHeightPx);
mProjectionCV.put(2, 0, 0.0);
mProjectionCV.put(2, 1, 0.0);
mProjectionCV.put(2, 2, 0.0);
}
return mProjectionCV;
}
}

然后是ImageDetectionFilter类：

public class ImageDetectionFilter implements ARFilter {

private final Mat mReferenceImage;
private final MatOfKeyPoint mReferenceKeypoints =
new MatOfKeyPoint();
private final Mat mReferenceDescriptors = new Mat();
// CVType defines the color depth, number of channels, and
// channel layout in the image.
private final Mat mReferenceCorners =
new Mat(4, 1, CvType.CV_32FC2);

private final MatOfKeyPoint mSceneKeypoints =
new MatOfKeyPoint();
private final Mat mSceneDescriptors = new Mat();

private final Mat mGraySrc = new Mat();
private final MatOfDMatch mMatches = new MatOfDMatch();

private final FeatureDetector mFeatureDetector =
FeatureDetector.create(FeatureDetector.STAR);
private final DescriptorExtractor mDescriptorExtractor =
DescriptorExtractor.create(DescriptorExtractor.FREAK);
private final DescriptorMatcher mDescriptorMatcher =
DescriptorMatcher.create(
DescriptorMatcher.BRUTEFORCE_HAMMING);

private final MatOfDouble mDistCoeffs = new MatOfDouble(
0.0, 0.0, 0.0, 0.0);

private final CameraProjectionAdapter mCameraProjectionAdapter;
private final MatOfDouble mRVec = new MatOfDouble();
private final MatOfDouble mTVec = new MatOfDouble();
private final MatOfDouble mRotation = new MatOfDouble();
private final float[] mGLPose = new float[16];

private boolean mTargetFound = false;

/**
* 构造方法，功能依然是初始化，以及对参考图像进行特征点的检测和描述
* 但是相对第四章中的内容，多了一个cameraProjectionAdapter的对象
* @param context
* @param referenceImageResourceID
* @param cameraProjectionAdapter
* @throws IOException
*/
public ImageDetectionFilter(final Context context,
final int referenceImageResourceID,
final CameraProjectionAdapter cameraProjectionAdapter)
throws IOException {
// 获取参考图像帧，可以修改这里设置自己的标
mReferenceImage = Utils.loadResource(context,
referenceImageResourceID,
Highgui.CV_LOAD_IMAGE_COLOR);

final Mat referenceImageGray = new Mat();
Imgproc.cvtColor(mReferenceImage, referenceImageGray,
Imgproc.COLOR_BGR2GRAY);
Imgproc.cvtColor(mReferenceImage, mReferenceImage,
Imgproc.COLOR_BGR2RGBA);

mReferenceCorners.put(0, 0,
new double[] {0.0, 0.0});
mReferenceCorners.put(1, 0,
new double[] {referenceImageGray.cols(), 0.0});
mReferenceCorners.put(2, 0,
new double[] {referenceImageGray.cols(),
referenceImageGray.rows()});
mReferenceCorners.put(3, 0,
new double[] {0.0, referenceImageGray.rows()});

mFeatureDetector.detect(referenceImageGray,
mReferenceKeypoints);
mDescriptorExtractor.compute(referenceImageGray,
mReferenceKeypoints, mReferenceDescriptors);

mCameraProjectionAdapter = cameraProjectionAdapter;
}

// 复写这个接口方法
// 根据是否有标志（Target）获取mGLPose
@Override
public float[] getGLPose() {
return (mTargetFound ? mGLPose : null);
}

/**
* 同样对实时获取的视频帧进行特征点的检测描述和匹配
* 多了一个findPose（）方法
*/
@Override
public void apply(final Mat src, final Mat dst) {
Imgproc.cvtColor(src, mGraySrc, Imgproc.COLOR_RGBA2GRAY);

mFeatureDetector.detect(mGraySrc, mSceneKeypoints);
mDescriptorExtractor.compute(mGraySrc, mSceneKeypoints,
mSceneDescriptors);
mDescriptorMatcher.match(mSceneDescriptors,
mReferenceDescriptors, mMatches);

findPose();
draw(src, dst);
}

/**
* 从方法名可以看出，这个就是核心算法了，估算出摄像头位姿
*/
private void findPose() {

List matchesList = mMatches.toList();
if (matchesList.size() < 4) {
// There are too few matches to find the pose.
return;
}

List referenceKeypointsList =
mReferenceKeypoints.toList();
List sceneKeypointsList =
mSceneKeypoints.toList();

// Calculate the max and min distances between keypoints.
double maxDist = 0.0;
double minDist = Double.MAX_VALUE;
for(DMatch match : matchesList) {
double dist = match.distance;
if (dist < minDist) {
minDist = dist;
}
if (dist > maxDist) {
maxDist = dist;
}
}

// The thresholds for minDist are chosen subjectively
// based on testing. The unit is not related to pixel
// distances; it is related to the number of failed tests
// for similarity between the matched descriptors.
if (minDist > 50.0) {
// The target is completely lost.
mTargetFound = false;
return;
} else if (minDist > 25.0) {
// The target is lost but maybe it is still close.
// Keep using any previously found pose.
return;
}

// Identify "good" keypoints based on match distance.
List goodReferencePointsList =
new ArrayList();
ArrayList goodScenePointsList =
new ArrayList();
double maxGoodMatchDist = 1.75 * minDist;
for(DMatch match : matchesList) {
if (match.distance < maxGoodMatchDist) {
Point point =
referenceKeypointsList.get(match.trainIdx).pt;
Point3 point3 = new Point3(point.x, point.y, 0.0);
goodReferencePointsList.add(point3);
goodScenePointsList.add(
sceneKeypointsList.get(match.queryIdx).pt);
}
}

if (goodReferencePointsList.size() < 4 ||
goodScenePointsList.size() < 4) {
// There are too few good points to find the pose.
return;
}

MatOfPoint3f goodReferencePoints = new MatOfPoint3f();
goodReferencePoints.fromList(goodReferencePointsList);

MatOfPoint2f goodScenePoints = new MatOfPoint2f();
goodScenePoints.fromList(goodScenePointsList);

/**
* 前面代码实现和第四章中相同，goodReferencePoints的类型为MatOfPoint3f，之前的是MatOfPoint2f
* 为什么呢？
* 主要是因为solvePnP方法，这个方法是使用PNP算法从3D-2D点之间的对应关系计算位姿矩阵
* 在这里，3D就是参考图像帧，2D就是视频帧，projection为摄像机内部参数，可通过标定计算，
* 也可以像本文中使用SDK检测并计算，mDistCoeffs为摄像头畸变，本文不考虑畸变情况
* 以上是输入参数，以下两个是输出参数：mRVec, mTVec
* mRVec为旋转矩阵
* mTVec为平移矩阵，这两个参数也是我们最终需要的
*/
MatOfDouble projection =
mCameraProjectionAdapter.getProjectionCV();
// 使用PNP算法计算位姿矩阵
Calib3d.solvePnP(goodReferencePoints, goodScenePoints,
projection, mDistCoeffs, mRVec, mTVec);

double[] rVecArray = mRVec.toArray();
rVecArray[1] *= -1.0;
rVecArray[2] *= -1.0;
mRVec.fromArray(rVecArray);

// 将旋转矩阵转换成旋转向量，
// 关于罗德里格斯变换，可以参阅这篇文章：http://blog.sina.com.cn/s/blog_5fb3f125010100hp.html
Calib3d.Rodrigues(mRVec, mRotation);
//
double[] tVecArray = mTVec.toArray();

// 将计算得出的结果转换为4*4的位姿矩阵，即摄像头位姿，就是我们最终需要的结果。
mGLPose[0] = (float)mRotation.get(0, 0)[0];
mGLPose[1] = (float)mRotation.get(1, 0)[0];
mGLPose[2] = (float)mRotation.get(2, 0)[0];
mGLPose[3] = 0f;
mGLPose[4] = (float)mRotation.get(0, 1)[0];
mGLPose[5] = (float)mRotation.get(1, 1)[0];
mGLPose[6] = (float)mRotation.get(2, 1)[0];
mGLPose[7] = 0f;
mGLPose[8] = (float)mRotation.get(0, 2)[0];
mGLPose[9] = (float)mRotation.get(1, 2)[0];
mGLPose[10] = (float)mRotation.get(2, 2)[0];
mGLPose[11] = 0f;
mGLPose[12] = (float)tVecArray[0];
mGLPose[13] = -(float)tVecArray[1];
mGLPose[14] = -(float)tVecArray[2];
mGLPose[15] = 1f;

mTargetFound = true;
}

// 当发现标志时，这里不再绘制边框，只在没有发现标的时候，在左上角绘制标志图片的缩略图
// 提示需要检测这样的图像（这个功能还蛮不错的）
protected void draw(Mat src, Mat dst) {

if (dst != src) {
src.copyTo(dst);
}

if (!mTargetFound) {
// The target has not been found.

// Draw a thumbnail of the target in the upper-left
// corner so that the user knows what it is.

int height = mReferenceImage.height();
int width = mReferenceImage.width();
int maxDimension = Math.min(dst.width(),
dst.height()) / 2;
double aspectRatio = width / (double)height;
if (height > width) {
height = maxDimension;
width = (int)(height * aspectRatio);
} else {
width = maxDimension;
height = (int)(width / aspectRatio);
}
Mat dstROI = dst.submat(0, height, 0, width);
Imgproc.resize(mReferenceImage, dstROI, dstROI.size(),
0.0, 0.0, Imgproc.INTER_AREA);
}
}
}

最后是ARCubeRenderer类，即OpenGL渲染类：

/**
* OpenGL的渲染类
* @author scy
*
*/
public class ARCubeRenderer implements GLSurfaceView.Renderer {

public ARFilter filter;
public CameraProjectionAdapter cameraProjectionAdapter;
public float scale = 100f;

private static final ByteBuffer VERTICES;
private static final ByteBuffer COLORS;
private static final ByteBuffer TRIANGLE_FAN_0;
private static final ByteBuffer TRIANGLE_FAN_1;

static {
VERTICES = ByteBuffer.allocateDirect(96);
VERTICES.order(ByteOrder.nativeOrder());
VERTICES.asFloatBuffer().put(new float[] {
-1f, 1f, 1f,
1f, 1f, 1f,
1f, -1f, 1f,
-1f, -1f, 1f,

-1f, 1f, -1f,
1f, 1f, -1f,
1f, -1f, -1f,
-1f, -1f, -1f
});
VERTICES.position(0);

COLORS = ByteBuffer.allocateDirect(32);
COLORS.put(new byte[] {
Byte.MAX_VALUE, Byte.MAX_VALUE, 0, Byte.MAX_VALUE, // yellow
0, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, // cyan
0, 0, 0, Byte.MAX_VALUE, // black
Byte.MAX_VALUE, 0, Byte.MAX_VALUE, Byte.MAX_VALUE, // magenta

Byte.MAX_VALUE, 0, 0, Byte.MAX_VALUE, // red
0, Byte.MAX_VALUE, 0, Byte.MAX_VALUE, // green
0, 0, Byte.MAX_VALUE, Byte.MAX_VALUE, // blue
0, 0, 0, Byte.MAX_VALUE // black
});
COLORS.position(0);

TRIANGLE_FAN_0 = ByteBuffer.allocate(18);
TRIANGLE_FAN_0.put(new byte[] {
1, 0, 3,
1, 3, 2,
1, 2, 6,
1, 6, 5,
1, 5, 4,
1, 4, 0
});
TRIANGLE_FAN_0.position(0);

TRIANGLE_FAN_1 = ByteBuffer.allocate(18);
TRIANGLE_FAN_1.put(new byte[] {
7, 4, 5,
7, 5, 6,
7, 6, 2,
7, 2, 3,
7, 3, 0,
7, 0, 4
});
TRIANGLE_FAN_1.position(0);
}

@Override
public void onDrawFrame(final GL10 gl) {

gl.glClear(GL10.GL_COLOR_BUFFER_BIT |
GL10.GL_DEPTH_BUFFER_BIT);
gl.glClearColor(0f, 0f, 0f, 0f); // transparent

if (filter == null) {
return;
}

if (cameraProjectionAdapter == null) {
return;
}
// 获取摄像头位姿矩阵
float[] pose = filter.getGLPose();
if (pose == null) {
return;
}

/**
* 这里有两个作用，一是设置投影矩阵
* 二是模型试图矩阵，注意这两个顺序不能乱，
* 然后绘制三维模型
*/
gl.glMatrixMode(GL10.GL_PROJECTION);
float[] projection =
cameraProjectionAdapter.getProjectionGL();
gl.glLoadMatrixf(projection, 0);

gl.glMatrixMode(GL10.GL_MODELVIEW);
gl.glLoadMatrixf(pose, 0);
gl.glTranslatef(0f, 0f, 1f);
gl.glScalef(scale, scale, scale);

// 开始绘制三维虚拟模型，这里是OpenGL的内容，网上有很多介绍
// Android OpenGL开发的教程，有兴趣可以去看看，这里我就在不再赘述了。
gl.glEnableClientState(GL10.GL_VERTEX_ARRAY);
gl.glEnableClientState(GL10.GL_COLOR_ARRAY);

gl.glVertexPointer(3, GL11.GL_FLOAT, 0, VERTICES);
gl.glColorPointer(4, GL11.GL_UNSIGNED_BYTE, 0, COLORS);

gl.glDrawElements(GL10.GL_TRIANGLE_FAN, 18,
GL10.GL_UNSIGNED_BYTE, TRIANGLE_FAN_0);
gl.glDrawElements(GL10.GL_TRIANGLE_FAN, 18,
GL10.GL_UNSIGNED_BYTE, TRIANGLE_FAN_1);
}

@Override
public void onSurfaceChanged(final GL10 gl, final int width,
final int height) {
}

@Override
public void onSurfaceCreated(final GL10 arg0,
final EGLConfig config) {
}
}

附图：

^_^本团队专业从事移动增强现实应用开发以及解决方案，有合作请私信联系！^_^

搜索历史

OpenCV&OpenGL开发AR的实现

相关推荐

精选推荐

最新话题

热门话题

创建小组步骤

站长推荐 /6