【EASY EAI Nano人工智能开发套件试用体验】摄像头驱动程序-carmera库、display库使用-opencv和v4l2测试

仓库

VID_20230625_222439

opencv输入测试

直接使用 opencv VideoCapture 读取摄像头：

cv::VideoCapture capture;
capture.open(vid);
cv::Mat frame;

double rate = capture.get(cv::CAP_PROP_POS_FRAMES); // 获取视频文件的帧率
int delay = cvRound(1000.000 / rate);				// 延时参数

// while (cv::waitKey(30) != 'q')
while (1)
{
	/*采用>>的方式读入视频*/
	capture >> frame;
	if (frame.empty())
		break;

	/*imshow()在窗口中显示*/
	cv::imshow("读取摄像头", frame);
	/*WaitKey()控制帧率*/
	cv::waitKey(30);
}

在cmakelist文件加入opencv的库支持：

file(GLOB OPENCV_LIBS "/mnt/usr/lib/arm-linux-gnueabihf/libopencv*.so")
TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${OPENCV_LIBS})

报错：

[ WARN:0] global ../modules/videoio/src/cap_gstreamer.cpp (480) isPipelinePlaying OpenCV | GStreamer warning: GStreamer: pipeline have not been created
[ WARN:0] global ../modules/videoio/src/cap_v4l.cpp (887) open VIDEOIO(V4L2:/dev/video0): can't open camera by index

根据报错信息，这些摄像头设备应该是无法像uvc摄像头那样直接通过 index 打开读取的。

v4l2方式读取

使用 v4l2-ctl 抓取：

v4l2-ctl -d /dev/video0  --try-fmt-video=width=1920,height=1080,pixelformat=NV12 --stream-mmap=3 --stream-to=/tmp/test.yuv --stream-count=10 --stream-poll

ffplay -f rawvideo -video_size 1920x1080 -pix_fmt nv12 test.yuv

发现依然是一片花屏绿幕

使用 v4l2 命令是可以捕获图像的，猜测 v4l2 方式可以打开摄像头

// 1-设备的打开和关
char videoName[100] = "/dev/video0";
sprintf(videoName, "/dev/video%d", vid);
int fd = open(videoName, O_RDWR); // 打开设备
struct v4l2_capability cap;

// 2-查询设备属
ioctl(fd, VIDIOC_QUERYCAP, &cap);
printf("DriverName\t%s\r\nCard Name\t%s\r\nBus info\t%s\r\nDriverVersion\t%u.%u.%u\r\n",
	   cap.driver, cap.card, cap.bus_info, (cap.version >> 16) & 0XFF, (cap.version >> 8) & 0XFF, (cap.version) & 0xFF);

// 3-帧格-显示所有支持的格式
struct v4l2_fmtdesc fmtdesc;
unsigned int min = 0;

fmtdesc.index = 0;
fmtdesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
printf("Supportformat:\r\n");
while (ioctl(fd, VIDIOC_ENUM_FMT, &fmtdesc) != -1)
{
	printf("\t%d.%s\r\n", fmtdesc.index + 1, fmtdesc.description);
	fmtdesc.index++;
}

// 4-显示当前帧的相关信息
struct v4l2_format fmt;
fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;

fmt.fmt.pix.width = 1080;
fmt.fmt.pix.height = 1920;
fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
fmt.fmt.pix.field = V4L2_FIELD_NONE;

ioctl(fd, VIDIOC_S_FMT, &fmt);
ioctl(fd, VIDIOC_G_FMT, &fmt);

printf("Currentdata format information:\r\n\twidth:%d\r\n\theight:%d\r\n", fmt.fmt.pix.width, fmt.fmt.pix.height);
//    struct v4l2_fmtdesc fmtdesc;
fmtdesc.index = 1;
fmtdesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
while (ioctl(fd, VIDIOC_ENUM_FMT, &fmtdesc) != -1)
{
	if (fmtdesc.pixelformat & fmt.fmt.pix.pixelformat)
	{
		printf("\tformat:%s\n", fmtdesc.description);
		break;
	}
	fmtdesc.index++;
}

//    init_device(fd);

// 1-申请一个拥有四个缓冲帧的缓冲区
struct v4l2_requestbuffers req;
req.count = n_buffers;
req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
req.memory = V4L2_MEMORY_MMAP;
ioctl(fd, VIDIOC_REQBUFS, &req);

buffers = (buffer *)calloc(req.count, sizeof(*buffers));
if (!buffers)
{
	fprintf(stderr, "Out of memory\n");
	exit(EXIT_FAILURE);
}

// 2-映射内存
for (unsigned int n_buffers = 0; n_buffers < req.count; ++n_buffers)
{
	struct v4l2_buffer buf;
	memset(&buf, 0, sizeof(buf));
	buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
	buf.memory = V4L2_MEMORY_MMAP;
	buf.index = n_buffers;
	// 查询序号为n_buffers 的缓冲区，得到其起始物理地址和大�??
	if (-1 == ioctl(fd, VIDIOC_QUERYBUF, &buf))
		exit(-1);
	buffers[n_buffers].length = buf.length;
	// 映射内存
	buffers[n_buffers].start =
		mmap(
			NULL,
			buf.length,
			PROT_READ | PROT_WRITE,
			MAP_SHARED,
			fd,
			buf.m.offset);
	if (MAP_FAILED == buffers[n_buffers].start)
		exit(-1);
}
//    init_mmap(fd);

// 3-把四个缓冲帧放入队列，并启动数据�??
unsigned int i;
enum v4l2_buf_type type;
// 将缓冲帧放入队列
for (i = 0; i < n_buffers; ++i)
{
	struct v4l2_buffer buf;
	buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
	buf.memory = V4L2_MEMORY_MMAP;
	buf.index = i;
	ioctl(fd, VIDIOC_QBUF, &buf);
	//        usleep(300000);
}
type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ioctl(fd, VIDIOC_STREAMON, &type);

cv::VideoWriter Writer("dect_result2.avi", cv::VideoWriter::fourcc('D', 'I', 'V', 'X'), 30, cv::Size(640, 480), true);
while (cv::waitKey(30) != 'q')
{
	struct v4l2_buffer buf;
	//    clear (buf);
	buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
	buf.memory = V4L2_MEMORY_MMAP;
	// 从缓冲区取出一个缓冲帧
	ioctl(fd, VIDIOC_DQBUF, &buf);
	// 图像处理
	// printf("bytesused:%d\n", buf.bytesused);
	// printf("buffers length:%d\n", buffers->length);
	// printf("buf length:%d\n", buf.length);
	// printf("memory:%d\n", buf.memory);
	// printf("m.userptr:%d\n\n", buf.m.userptr);
	// printf("m.offset:%d\n\n", buf.m.offset);
	// process_image(buffers[buf.index].start, buf.bytesused);

	cv::Mat image(480, 640, CV_8UC2, (unsigned char *)buffers[buf.index].start);
	cv::Mat img2;
	cv::cvtColor(image, img2, cv::COLOR_YUV2BGR_YUYV);
	cv::imshow("current", img2);
	// Writer.write(img2);

	// 将取出的缓冲帧放回缓冲区
	ioctl(fd, VIDIOC_QBUF, &buf);
	// usleep(300000);
}

Writer.release();

close(fd); // 关闭设备

但依然失败：

DriverName      rkcif
Card Name       rkcif
Bus info        platform:rkcif_mipi_lvds
DriverVersion   4.19.111
Supportformat:
Currentdata format information:
		width:1080
		height:1920

库方式读取

查询了rv1126 isp相关资料后发现，mipi及isp相关驱动应该是闭源，可能需要签nda的批量用户才能获取，easy-eai也采用.a .so封装，应该是不予开放，因此必须使用 easy-eai 的库驱动：

int ret = 0;

	char *pbuf = NULL;
	int skip = 0;

	pthread_t mTid;
	// Result_t Result;

	cv::Mat algorithm_image;
	pthread_mutex_t img_lock;
	cv::Mat image;
	// 1.打开摄像头
#define CAMERA_WIDTH 720
#define CAMERA_HEIGHT 1280
#define IMGRATIO 3
#define IMAGE_SIZE (CAMERA_WIDTH * CAMERA_HEIGHT * IMGRATIO)
	ret = rgbcamera_init(CAMERA_WIDTH, CAMERA_HEIGHT, 90);
	if (ret)
	{
		printf("error: %s, %d\n", __func__, __LINE__);
		goto exit4;
	}

	pbuf = NULL;
	pbuf = (char *)malloc(IMAGE_SIZE);
	if (!pbuf)
	{
		printf("error: %s, %d\n", __func__, __LINE__);
		ret = -1;
		goto exit3;
	}

	// 跳过前10帧
	skip = 10;
	while (skip--)
	{
		ret = rgbcamera_getframe(pbuf);
		if (ret)
		{
			printf("error: %s, %d\n", __func__, __LINE__);
			goto exit2;
		}
	}

	// 2.创建识别线程，以及图像互斥锁
	pthread_mutex_init(&img_lock, NULL);
	// Result.person_number = 0;
	// CreateNormalThread(detect_thread_entry, &Result, &mTid);

	// 3.显示初始化
#define SCREEN_WIDTH 720
#define SCREEN_HEIGHT 1280
	ret = disp_init(SCREEN_WIDTH, SCREEN_HEIGHT);
	if (ret)
	{
		printf("error: %s, %d\n", __func__, __LINE__);
		goto exit1;
	}

	// 4.(取流 + 显示)循环
	while (1)
	{
		// 4.1、取流
		pthread_mutex_lock(&img_lock);
		ret = rgbcamera_getframe(pbuf);
		if (ret)
		{
			printf("error: %s, %d\n", __func__, __LINE__);
			pthread_mutex_unlock(&img_lock);
			continue;
		}
		algorithm_image = cv::Mat(CAMERA_HEIGHT, CAMERA_WIDTH, CV_8UC3, pbuf);
		image = algorithm_image.clone();
		pthread_mutex_unlock(&img_lock);
		// 4.2、显示
		disp_commit(image.data, IMAGE_SIZE);

		usleep(20 * 1000);
	}

exit1:
	pthread_mutex_destroy(&img_lock);
exit2:
	free(pbuf);
	pbuf = NULL;
exit3:
	rgbcamera_exit();
exit4:
	return ret;

cmakelist 文件加入 easy eai 和 rockchip 支持

SET(MYSYSSDKLIB "/mnt/usr/lib")
SET(toolkit_root /home/developer/project/demo/EASY-EAI-Toolkit-C-Solution/easyeai-api)

# 1 库目录

# SYS SDK
LINK_DIRECTORIES(${MYSYSSDKLIB})

# EASY EAI
LINK_DIRECTORIES(${toolkit_root}/common_api/system_opt)
LINK_DIRECTORIES(${toolkit_root}/peripheral_api/camera)
LINK_DIRECTORIES(${toolkit_root}/peripheral_api/display)

# 2 库头文件
INCLUDE_DIRECTORIES(
	${toolkit_root}/peripheral_api/camera
	${toolkit_root}/peripheral_api/display
)

# 3 .lib .so
TARGET_LINK_LIBRARIES(${PROJECT_NAME}
	${toolkit_root}/peripheral_api/display/libdisplay.a		pthread
	${toolkit_root}/peripheral_api/camera/libcamera.a		easymedia rga
	${toolkit_root}/common_api/system_opt/libsystem_opt.a	easymedia rga rkaiq
)

其中尤其注意，除了 SDK EASY-EAI-Toolkit-C-Solution 里的libdisplay.a等几个库，还有库里面的 easymedia rga rkaiq；
- easymedia 应该是 easy eai 适配的摄像头驱动库；
- rga rkaiq 是瑞芯微 isp 类的驱动库
- libcamera.a 这些就是 easy eai 适配的摄像头应用库；
- libdisplay.a 是屏幕驱动库，如果使用Qt应该就不需要走这个库。
程序：仓库